nohi191212 commited on
Commit
cde28d7
·
verified ·
1 Parent(s): 09c5871

Publish latest paper, paired outcomes and verified reproduction materials

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +6 -0
  2. releases/20260913_lr/LATEST_RELEASE.md +50 -0
  3. releases/20260913_lr/MODELS_AND_DATA.md +51 -0
  4. releases/20260913_lr/REPRODUCE.md +50 -0
  5. releases/20260913_lr/appendix.pdf +3 -0
  6. releases/20260913_lr/main.pdf +3 -0
  7. releases/20260913_lr/paired_outcomes.tar.gz +3 -0
  8. releases/20260913_lr/paper_data/budget_diagnostics.json +1 -0
  9. releases/20260913_lr/paper_data/comparisons.json +0 -0
  10. releases/20260913_lr/paper_data/cost_20260911/construction_router.json +27 -0
  11. releases/20260913_lr/paper_data/cost_20260911/cost_summary.json +1036 -0
  12. releases/20260913_lr/paper_data/cost_20260911/fixed_budget_cost_performance.csv +55 -0
  13. releases/20260913_lr/paper_data/cost_20260911/full_cost_performance_curves.csv +0 -0
  14. releases/20260913_lr/paper_data/cost_20260911/grefcoco_router.json +27 -0
  15. releases/20260913_lr/paper_data/cost_20260911/instancevg_specialist.json +134 -0
  16. releases/20260913_lr/paper_data/cost_20260911/make_cost_analysis_20260911.py +457 -0
  17. releases/20260913_lr/paper_data/cost_20260911/minicpm_grefcoco.json +385 -0
  18. releases/20260913_lr/paper_data/cost_20260911/minicpm_grefcoco_flops.json +117 -0
  19. releases/20260913_lr/paper_data/cost_20260911/qwen_construction.json +385 -0
  20. releases/20260913_lr/paper_data/cost_20260911/qwen_construction_flops.json +177 -0
  21. releases/20260913_lr/paper_data/cost_20260911/router_architecture.json +37 -0
  22. releases/20260913_lr/paper_data/cost_20260911/yolo26x_specialist.json +134 -0
  23. releases/20260913_lr/paper_data/cost_20260911/成本性能补充报告.md +69 -0
  24. releases/20260913_lr/paper_data/cost_components.json +41 -0
  25. releases/20260913_lr/paper_data/cost_five_seeds.json +3299 -0
  26. releases/20260913_lr/paper_data/earlier_results.json +0 -0
  27. releases/20260913_lr/paper_data/fig2_audit/remote_counts.json +74 -0
  28. releases/20260913_lr/paper_data/fig2_audit/remote_counts.py +25 -0
  29. releases/20260913_lr/paper_data/five_seed_counts.json +7 -0
  30. releases/20260913_lr/paper_data/full_budget_results.json +0 -0
  31. releases/20260913_lr/paper_data/historical_results_v44.json +0 -0
  32. releases/20260913_lr/paper_data/joint_MiniCPM_comparison_20260910.json +58 -0
  33. releases/20260913_lr/paper_data/joint_Qwen3.8_comparison_20260910.json +58 -0
  34. releases/20260913_lr/paper_data/layout_verification.json +29 -0
  35. releases/20260913_lr/paper_data/lr_control_curves.json +0 -0
  36. releases/20260913_lr/paper_data/lr_control_summary.json +156 -0
  37. releases/20260913_lr/paper_data/lr_main_summary.json +210 -0
  38. releases/20260913_lr/paper_data/main_table_cost_20260912/build_figures.py +101 -0
  39. releases/20260913_lr/paper_data/main_table_cost_20260912/components/instancevg_router.json +239 -0
  40. releases/20260913_lr/paper_data/main_table_cost_20260912/components/instancevg_specialist.json +254 -0
  41. releases/20260913_lr/paper_data/main_table_cost_20260912/components/minicpm_flops.json +261 -0
  42. releases/20260913_lr/paper_data/main_table_cost_20260912/components/minicpm_latency.json +2105 -0
  43. releases/20260913_lr/paper_data/main_table_cost_20260912/components/qwen_flops.json +341 -0
  44. releases/20260913_lr/paper_data/main_table_cost_20260912/components/qwen_latency.json +0 -0
  45. releases/20260913_lr/paper_data/main_table_cost_20260912/components/yolo26x_router.json +238 -0
  46. releases/20260913_lr/paper_data/main_table_cost_20260912/components/yolo26x_specialist.json +254 -0
  47. releases/20260913_lr/paper_data/main_table_cost_20260912/cost_compact.pdf +0 -0
  48. releases/20260913_lr/paper_data/main_table_cost_20260912/cost_compact.png +3 -0
  49. releases/20260913_lr/paper_data/main_table_cost_20260912/cost_compact_verified.png +0 -0
  50. releases/20260913_lr/paper_data/main_table_cost_20260912/cost_summary.json +75 -0
.gitattributes CHANGED
@@ -49,3 +49,9 @@ paper_results/figures/Fig1_editable.pptx filter=lfs diff=lfs merge=lfs -text
49
  paper_results/figures/flops_performance.png filter=lfs diff=lfs merge=lfs -text
50
  paper_results/figures/latency_performance.png filter=lfs diff=lfs merge=lfs -text
51
  paper_results/figures/method.pdf filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
49
  paper_results/figures/flops_performance.png filter=lfs diff=lfs merge=lfs -text
50
  paper_results/figures/latency_performance.png filter=lfs diff=lfs merge=lfs -text
51
  paper_results/figures/method.pdf filter=lfs diff=lfs merge=lfs -text
52
+ releases/20260913_lr/paper_data/main_table_cost_20260912/cost_compact.png filter=lfs diff=lfs merge=lfs -text
53
+ releases/20260913_lr/paper_data/main_table_cost_qwen_20260912/cost_compact.png filter=lfs diff=lfs merge=lfs -text
54
+ releases/20260913_lr/paper_data/reproduction/construction_image_path.jpg filter=lfs diff=lfs merge=lfs -text
55
+ releases/20260913_lr/paper_data/reproduction/nlvr2_image2.jpg filter=lfs diff=lfs merge=lfs -text
56
+ releases/20260913_lr/paper_data/test_results.json filter=lfs diff=lfs merge=lfs -text
57
+ releases/20260913_lr/paper_data/validation_manifest.json filter=lfs diff=lfs merge=lfs -text
releases/20260913_lr/LATEST_RELEASE.md ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Latest release: 20260913_lr
2
+
3
+ The release contains the latest paper results and the full experiment record. Download artifacts from `nohi191212/ModelCollaboration` on Hugging Face under `releases/20260913_lr/`.
4
+
5
+ ## Weights and records
6
+
7
+ `full_budget_20260913_weights.tar.gz` contains exactly **80 router checkpoints**: 16 model pairs × five seeds for the main table. This release publishes no other router weights.
8
+
9
+ The study-specific `_records.tar.gz` archives retain configurations, training histories, validation/test outputs, curves and selection records for the experiments. Extract them into the same artifact directory; paths under `outputs/` are preserved. Earlier fixed-answer CSS/OvA adaptations are separate from the corrected K-class implementations.
10
+
11
+ Previously released backbone and specialist weights remain in the root-level `weights.tar`. External models and datasets are linked in `MODELS_AND_DATA.md`.
12
+
13
+ ## Data
14
+
15
+ Eight `training_inputs_<expert>.tar.gz` files contain the train/validation/test arrays and example records for the latest 8M frozen-backbone setup. They contain image/text features, the selected specialist hidden state, confidence, output features, validity masks and labels. `input_manifest.json` records dimensions, sample counts and hidden-layer names. Set `ICASSP_DATA_ROOT` to the directory containing `training_inputs/` and this manifest.
16
+
17
+ `paired_outcomes.tar.gz` provides the paired correctness and task-metric sufficient statistics used for result replay. `paper_replay_runs.tar.gz` contains the 400 training runs used by the latest Tables 1 and 3, with raw logits and configurations. These two files are sufficient for `scripts/replay_latest_results.py`.
18
+
19
+ `additional_analysis_records.tar.gz` contains scoring explorations and cost records. `experiment_sources.tar.gz` preserves original experiment source snapshots, also included in the GitHub repository.
20
+
21
+ The latest `paper/data/` directory stores table and figure summaries. Main weights were selected over the full 0–100% validation curve. Table 3 retains its earlier training selection setting, as specified in the supplement and run configurations.
22
+
23
+ ## Training examples
24
+
25
+ Install the dependencies in `requirements.txt` and a PyTorch build for your GPU. Commands below are run from the repository root. They start training only when invoked.
26
+
27
+ For normalized LR, download/extract a specialist's input archive and `input_manifest.json`, then run:
28
+
29
+ ```bash
30
+ export ICASSP_DATA_ROOT="$PWD/artifacts/releases/20260913_lr"
31
+ python code/experiments/full_budget_train.py --source configs/full_budget/full_cub_MiniCPM_s2026 --output outputs/my_lr_run
32
+ ```
33
+
34
+ For post-hoc deferral, extract its records archive, choose a saved `config.json`, and run:
35
+
36
+ ```bash
37
+ python code/experiments/train_ltd.py --root "$PWD" --config "$ICASSP_DATA_ROOT/outputs/ltd_fixed_backbone_20260913/runs/cub_MiniCPM_posthoc_c0_s2026/config.json" --output outputs/my_posthoc_run --device cuda
38
+ ```
39
+
40
+ Use `train_three.py` for a configuration from `three_state_20260913`, and `train_native.py` for a configuration from `ltd_kclass_native_20260913`. Configurations are supplied in the records and weight archives. The original objective experiment sources are in `experiments/source_snapshots/`; the five-seed objective trainer is also in `code/objectives/`.
41
+
42
+ To evaluate a newly trained post-hoc/three-state/native run, use its corresponding `evaluate_ltd.py`, `evaluate_three.py` or `evaluate_native.py` with `--help` for arguments. Input paths are resolved through `ICASSP_DATA_ROOT`.
43
+
44
+ Earlier architecture studies use other feature sizes or hidden layers. Their original configurations and source snapshots are supplied; regenerating those inputs from images uses the original feature-preparation scripts and the upstream datasets. The cached inputs are the latest common 8M setup.
45
+
46
+ ## Verification
47
+
48
+ `scripts/replay_latest_results.py` recomputes the latest Tables 1 and 3 using the saved per-example outputs. `verification_latest.json` records 80 main runs, 320 Table 3 runs and 19,392 checked curve points. `weight_reload_verification.json` checks all 80 main weights against saved logits using the released cache loader (128 validation inputs per run).
49
+
50
+ Original code is MIT licensed. Derived data retain the licenses of their source datasets; consult `MODELS_AND_DATA.md` before redistribution.
releases/20260913_lr/MODELS_AND_DATA.md ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Upstream models and datasets
2
+
3
+ ### Vision-language models and router teacher
4
+
5
+ | Model | Hugging Face download | Role |
6
+ |---|---|---|
7
+ | MiniCPM-V-4.5 | [openbmb/MiniCPM-V-4_5](https://huggingface.co/openbmb/MiniCPM-V-4_5) | VLM in the main comparisons |
8
+ | Qwen3.8-27B-FP8 | [Qwen/Qwen3.8-27B-FP8](https://huggingface.co/Qwen/Qwen3.8-27B-FP8) | FP8 VLM in the main comparisons |
9
+ | SigLIP2 Base, patch 16, resolution 224 | [google/siglip2-base-patch16-224](https://huggingface.co/google/siglip2-base-patch16-224) | Teacher for the compact image/text encoders |
10
+
11
+ ```bash
12
+ hf download openbmb/MiniCPM-V-4_5 --local-dir models/MiniCPM-V-4_5
13
+ hf download Qwen/Qwen3.8-27B-FP8 --local-dir models/Qwen3.8-27B-FP8
14
+ hf download google/siglip2-base-patch16-224 --local-dir models/siglip2-base-patch16-224
15
+ ```
16
+
17
+ Follow each model card for serving dependencies. Preserve the Qwen FP8 variant and the task prompts in [`prompts/`](prompts/) to match the experiments.
18
+
19
+ ### Specialist models
20
+
21
+ | Specialist / task | Download or implementation | Checkpoint note |
22
+ |---|---|---|
23
+ | InstanceVG / gRefCOCO | [Hugging Face weights](https://huggingface.co/Dmmm997/InstanceVG/tree/main/gres/InstanceVG-grefcoco), [implementation](https://github.com/Dmmm1997/InstanceVG) | Use the gRefCOCO checkpoint |
24
+ | ViLT / NLVR2 | [dandelin/vilt-b32-finetuned-nlvr2](https://huggingface.co/dandelin/vilt-b32-finetuned-nlvr2) | Task-finetuned upstream weights |
25
+ | BEiT-3 / NLVR2 | [Raghavan/beit3_base_patch16_224_nlvr2](https://huggingface.co/Raghavan/beit3_base_patch16_224_nlvr2), [implementation](https://github.com/microsoft/unilm/tree/master/beit3) | Task-finetuned upstream weights; preserve the recorded input processing |
26
+ | Grounding DINO / gRefCOCO | [Implementation and checkpoints](https://github.com/IDEA-Research/GroundingDINO) | Match the experiment's architecture and checkpoint |
27
+ | ResNet-50 / CUB-200-2011 | [Torchvision ResNet-50](https://docs.pytorch.org/vision/stable/models/generated/torchvision.models.resnet50.html) | Initialization only; the CUB-trained checkpoint is also required |
28
+ | GLSim / CUB-200-2011 | Paper checkpoint: `cub_vit_b16_16_2.pth` | Exact public download URL pending |
29
+ | YOLO26x / ConstructionSite | [Our trained weights](https://huggingface.co/nohi191212/ModelCollaboration/resolve/main/weights.tar) | `specialists/yolo26x/best.pt` inside the archive |
30
+ | RT-DETR-X / ConstructionSite | [Our trained weights](https://huggingface.co/nohi191212/ModelCollaboration/resolve/main/weights.tar) | `specialists/rtdetr_x_fp32/best.pt`; recorded inference uses FP32 |
31
+
32
+ ### Datasets
33
+
34
+ | Dataset | Official download / access instructions | Required contents |
35
+ |---|---|---|
36
+ | CUB-200-2011 | [Caltech dataset record](https://data.caltech.edu/records/65de6-vp158) | Images, classes, bounding boxes and original split annotations |
37
+ | gRefCOCO | [Official annotations](https://github.com/henghuiding/gRefCOCO), [COCO images](https://cocodataset.org/#download) | Expressions, annotations and corresponding COCO images; follow upstream image-version instructions |
38
+ | NLVR2 | [Official dataset page](https://lil.nlp.cornell.edu/nlvr/), [download instructions](https://github.com/lil-lab/nlvr/tree/master/nlvr2) | Paired images, statements and labels |
39
+ | ConstructionSite (ConstructionSite10k) | [LouisChen15/ConstructionSite](https://huggingface.co/datasets/LouisChen15/ConstructionSite) | Images and annotations; accept the dataset's access conditions |
40
+
41
+ For ConstructionSite, after access is granted:
42
+
43
+ ```bash
44
+ hf auth login
45
+ hf download LouisChen15/ConstructionSite --repo-type dataset --local-dir data/constructionsite10k
46
+ ```
47
+
48
+ Dataset and model licenses remain those of their authors. ConstructionSite is distributed under CC BY-NC 4.0. Original dataset splits are not automatically the router train/validation/test splits: follow the split definitions, exclusions and evaluation rules in [`paper/supplementary.pdf`](paper/supplementary.pdf).
49
+
50
+
51
+ Upstream models and datasets retain their original licenses.
releases/20260913_lr/REPRODUCE.md ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Latest release: 20260913_lr
2
+
3
+ The release contains the latest paper results and the full experiment record. Download artifacts from `nohi191212/ModelCollaboration` on Hugging Face under `releases/20260913_lr/`.
4
+
5
+ ## Weights and records
6
+
7
+ `full_budget_20260913_weights.tar.gz` contains exactly **80 router checkpoints**: 16 model pairs × five seeds for the main table. This release publishes no other router weights.
8
+
9
+ The study-specific `_records.tar.gz` archives retain configurations, training histories, validation/test outputs, curves and selection records for the experiments. Extract them into the same artifact directory; paths under `outputs/` are preserved. Earlier fixed-answer CSS/OvA adaptations are separate from the corrected K-class implementations.
10
+
11
+ Previously released backbone and specialist weights remain in the root-level `weights.tar`. External models and datasets are linked in `MODELS_AND_DATA.md`.
12
+
13
+ ## Data
14
+
15
+ Eight `training_inputs_<expert>.tar.gz` files contain the train/validation/test arrays and example records for the latest 8M frozen-backbone setup. They contain image/text features, the selected specialist hidden state, confidence, output features, validity masks and labels. `input_manifest.json` records dimensions, sample counts and hidden-layer names. Set `ICASSP_DATA_ROOT` to the directory containing `training_inputs/` and this manifest.
16
+
17
+ `paired_outcomes.tar.gz` provides the paired correctness and task-metric sufficient statistics used for result replay. `paper_replay_runs.tar.gz` contains the 400 training runs used by the latest Tables 1 and 3, with raw logits and configurations. These two files are sufficient for `scripts/replay_latest_results.py`.
18
+
19
+ `additional_analysis_records.tar.gz` contains scoring explorations and cost records. `experiment_sources.tar.gz` preserves original experiment source snapshots, also included in the GitHub repository.
20
+
21
+ The latest `paper/data/` directory stores table and figure summaries. Main weights were selected over the full 0–100% validation curve. Table 3 retains its earlier training selection setting, as specified in the supplement and run configurations.
22
+
23
+ ## Training examples
24
+
25
+ Install the dependencies in `requirements.txt` and a PyTorch build for your GPU. Commands below are run from the repository root. They start training only when invoked.
26
+
27
+ For normalized LR, download/extract a specialist's input archive and `input_manifest.json`, then run:
28
+
29
+ ```bash
30
+ export ICASSP_DATA_ROOT="$PWD/artifacts/releases/20260913_lr"
31
+ python code/experiments/full_budget_train.py --source configs/full_budget/full_cub_MiniCPM_s2026 --output outputs/my_lr_run
32
+ ```
33
+
34
+ For post-hoc deferral, extract its records archive, choose a saved `config.json`, and run:
35
+
36
+ ```bash
37
+ python code/experiments/train_ltd.py --root "$PWD" --config "$ICASSP_DATA_ROOT/outputs/ltd_fixed_backbone_20260913/runs/cub_MiniCPM_posthoc_c0_s2026/config.json" --output outputs/my_posthoc_run --device cuda
38
+ ```
39
+
40
+ Use `train_three.py` for a configuration from `three_state_20260913`, and `train_native.py` for a configuration from `ltd_kclass_native_20260913`. Configurations are supplied in the records and weight archives. The original objective experiment sources are in `experiments/source_snapshots/`; the five-seed objective trainer is also in `code/objectives/`.
41
+
42
+ To evaluate a newly trained post-hoc/three-state/native run, use its corresponding `evaluate_ltd.py`, `evaluate_three.py` or `evaluate_native.py` with `--help` for arguments. Input paths are resolved through `ICASSP_DATA_ROOT`.
43
+
44
+ Earlier architecture studies use other feature sizes or hidden layers. Their original configurations and source snapshots are supplied; regenerating those inputs from images uses the original feature-preparation scripts and the upstream datasets. The cached inputs are the latest common 8M setup.
45
+
46
+ ## Verification
47
+
48
+ `scripts/replay_latest_results.py` recomputes the latest Tables 1 and 3 using the saved per-example outputs. `verification_latest.json` records 80 main runs, 320 Table 3 runs and 19,392 checked curve points. `weight_reload_verification.json` checks all 80 main weights against saved logits using the released cache loader (128 validation inputs per run).
49
+
50
+ Original code is MIT licensed. Derived data retain the licenses of their source datasets; consult `MODELS_AND_DATA.md` before redistribution.
releases/20260913_lr/appendix.pdf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b5bfda42c8b67bb5b356d1d316a9f48f9c4466019482dc232eb0017049236119
3
+ size 1090203
releases/20260913_lr/main.pdf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c5a140a24780ff19e84d870579fc094ab7b8c76296e123f1ab64d2eb52feba38
3
+ size 4826693
releases/20260913_lr/paired_outcomes.tar.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5bc2d10e03e7124ff1a073120cfc1e7951cbb4d8ef016b5a305c30a4c78143fd
3
+ size 2668397
releases/20260913_lr/paper_data/budget_diagnostics.json ADDED
@@ -0,0 +1 @@
 
 
1
+ [{"key": ["cub", "cub", "MiniCPM"], "areas": {"confidence": {"validation_fixed_A25": 0.9058333333333335, "test_fixed_A25": 0.6472903003106663, "test_ranked_A25_diagnostic": 0.7895926820849155, "selected_validation_actual_calls": 0.0, "selected_test_actual_calls": 0.0}, "learned": {"validation_fixed_A25": 0.9740333333333333, "test_fixed_A25": 0.8063444943044529, "test_ranked_A25_diagnostic": 0.8089368312046945, "selected_validation_actual_calls": 0.004, "selected_test_actual_calls": 0.0059371763893683125}}}, {"key": ["cub", "cub", "Qwen3.8"], "areas": {"confidence": {"validation_fixed_A25": 0.9171666666666667, "test_fixed_A25": 0.7316189161201244, "test_ranked_A25_diagnostic": 0.8101484294097343, "selected_validation_actual_calls": 0.0, "selected_test_actual_calls": 0.0}, "learned": {"validation_fixed_A25": 0.9834833333333335, "test_fixed_A25": 0.8129651363479462, "test_ranked_A25_diagnostic": 0.8117915084570246, "selected_validation_actual_calls": 0.0, "selected_test_actual_calls": 0.0}}}, {"key": ["cub", "glsim", "MiniCPM"], "areas": {"confidence": {"validation_fixed_A25": 0.9005000000000001, "test_fixed_A25": 0.7253365550569554, "test_ranked_A25_diagnostic": 0.8519848118743528, "selected_validation_actual_calls": 0.0, "selected_test_actual_calls": 0.0}, "learned": {"validation_fixed_A25": 0.9781666666666666, "test_fixed_A25": 0.8896064894718676, "test_ranked_A25_diagnostic": 0.8920262340352089, "selected_validation_actual_calls": 0.004, "selected_test_actual_calls": 0.008491542975491888}}}, {"key": ["cub", "glsim", "Qwen3.8"], "areas": {"confidence": {"validation_fixed_A25": 0.9213333333333334, "test_fixed_A25": 0.7860545391784605, "test_ranked_A25_diagnostic": 0.8691232309285468, "selected_validation_actual_calls": 0.0, "selected_test_actual_calls": 0.0}, "learned": {"validation_fixed_A25": 0.9858, "test_fixed_A25": 0.8977666551605109, "test_ranked_A25_diagnostic": 0.897169485674836, "selected_validation_actual_calls": 0.0, "selected_test_actual_calls": 0.0}}}, {"key": ["grefcoco", "groundingdino", "MiniCPM"], "areas": {"confidence": {"validation_fixed_A25": 0.5566729917773561, "test_fixed_A25": 0.27538212857669514, "test_ranked_A25_diagnostic": 0.2786603522105323, "selected_validation_actual_calls": 1.0, "selected_test_actual_calls": 1.0}, "learned": {"validation_fixed_A25": 0.6126923887834703, "test_fixed_A25": 0.37841703768822854, "test_ranked_A25_diagnostic": 0.3486317102912402, "selected_validation_actual_calls": 0.4579520697167756, "selected_test_actual_calls": 0.5818676800045374}}}, {"key": ["grefcoco", "groundingdino", "Qwen3.8"], "areas": {"confidence": {"validation_fixed_A25": 0.5648886077728582, "test_fixed_A25": 0.28625187873975555, "test_ranked_A25_diagnostic": 0.29324220854720245, "selected_validation_actual_calls": 1.0, "selected_test_actual_calls": 1.0}, "learned": {"validation_fixed_A25": 0.647486119895987, "test_fixed_A25": 0.42089272041516607, "test_ranked_A25_diagnostic": 0.3745852593369821, "selected_validation_actual_calls": 0.7759786351816713, "selected_test_actual_calls": 0.8846666477611093}}}, {"key": ["grefcoco", "instancevg", "MiniCPM"], "areas": {"confidence": {"validation_fixed_A25": 0.7465176751704266, "test_fixed_A25": 0.6639253608598249, "test_ranked_A25_diagnostic": 0.6590023537418825, "selected_validation_actual_calls": 0.6499402628434886, "selected_test_actual_calls": 0.5209426310864078}, "learned": {"validation_fixed_A25": 0.7538997821350764, "test_fixed_A25": 0.678765278053484, "test_ranked_A25_diagnostic": 0.6785259336982106, "selected_validation_actual_calls": 0.14395951929158762, "selected_test_actual_calls": 0.19038085245157815}}}, {"key": ["grefcoco", "instancevg", "Qwen3.8"], "areas": {"confidence": {"validation_fixed_A25": 0.7477616136060159, "test_fixed_A25": 0.6640728242066757, "test_ranked_A25_diagnostic": 0.6611405722712191, "selected_validation_actual_calls": 0.7099585353854804, "selected_test_actual_calls": 0.6838612710206166}, "learned": {"validation_fixed_A25": 0.7825750228406776, "test_fixed_A25": 0.7278393783852765, "test_ranked_A25_diagnostic": 0.7134446870657629, "selected_validation_actual_calls": 0.45595614589921996, "selected_test_actual_calls": 0.6200153135014037}}}, {"key": ["nlvr2", "nlvr2", "MiniCPM"], "areas": {"confidence": {"validation_fixed_A25": 0.8284302492122602, "test_fixed_A25": 0.8339025615268711, "test_ranked_A25_diagnostic": 0.835882901628758, "selected_validation_actual_calls": 0.8587797192781438, "selected_test_actual_calls": 0.8580038745784602}, "learned": {"validation_fixed_A25": 0.8353480378115155, "test_fixed_A25": 0.8422214249838561, "test_ranked_A25_diagnostic": 0.8411638085671236, "selected_validation_actual_calls": 0.9139215124606132, "selected_test_actual_calls": 0.9110425486116093}}}, {"key": ["nlvr2", "nlvr2", "Qwen3.8"], "areas": {"confidence": {"validation_fixed_A25": 0.8312804354053279, "test_fixed_A25": 0.8343976465523427, "test_ranked_A25_diagnostic": 0.8361340317141422, "selected_validation_actual_calls": 0.8587797192781438, "selected_test_actual_calls": 0.8580038745784602}, "learned": {"validation_fixed_A25": 0.8401432254368376, "test_fixed_A25": 0.8425816172777499, "test_ranked_A25_diagnostic": 0.8422458204778647, "selected_validation_actual_calls": 0.8419077628186766, "selected_test_actual_calls": 0.8434096290449882}}}, {"key": ["nlvr2", "vilt", "MiniCPM"], "areas": {"confidence": {"validation_fixed_A25": 0.7926095674591808, "test_fixed_A25": 0.7987371744277822, "test_ranked_A25_diagnostic": 0.8005237856066586, "selected_validation_actual_calls": 0.9598968776854769, "selected_test_actual_calls": 0.9593169261677549}, "learned": {"validation_fixed_A25": 0.7940532798625036, "test_fixed_A25": 0.8014909951926528, "test_ranked_A25_diagnostic": 0.8024137188778073, "selected_validation_actual_calls": 0.9819535949584648, "selected_test_actual_calls": 0.9805984071177442}}}, {"key": ["nlvr2", "vilt", "Qwen3.8"], "areas": {"confidence": {"validation_fixed_A25": 0.7940561443712404, "test_fixed_A25": 0.7988735021884195, "test_ranked_A25_diagnostic": 0.8006242376408124, "selected_validation_actual_calls": 0.9598968776854769, "selected_test_actual_calls": 0.9593169261677549}, "learned": {"validation_fixed_A25": 0.7962102549412775, "test_fixed_A25": 0.8036521489560164, "test_ranked_A25_diagnostic": 0.8044127143574658, "selected_validation_actual_calls": 0.9959896877685477, "selected_test_actual_calls": 0.9961684724115664}}}, {"key": ["construction", "yolo26x", "MiniCPM"], "areas": {"confidence": {"validation_fixed_A25": 0.1745164052515915, "test_fixed_A25": 0.17697501725814355, "test_ranked_A25_diagnostic": 0.1716572124279524, "selected_validation_actual_calls": 0.9386590584878745, "selected_test_actual_calls": 0.9254327563249002}, "learned": {"validation_fixed_A25": 0.2280241031947047, "test_fixed_A25": 0.21383535478219448, "test_ranked_A25_diagnostic": 0.21318248319590255, "selected_validation_actual_calls": 0.4573466476462197, "selected_test_actual_calls": 0.4531957390146471}}}, {"key": ["construction", "yolo26x", "Qwen3.8"], "areas": {"confidence": {"validation_fixed_A25": 0.17942553980644182, "test_fixed_A25": 0.18055274894913717, "test_ranked_A25_diagnostic": 0.1793307821359281, "selected_validation_actual_calls": 0.9286733238231099, "selected_test_actual_calls": 0.9221038615179761}, "learned": {"validation_fixed_A25": 0.2325252201898847, "test_fixed_A25": 0.21172533739102792, "test_ranked_A25_diagnostic": 0.21128777332230012, "selected_validation_actual_calls": 0.45135520684736086, "selected_test_actual_calls": 0.44766977363515315}}}, {"key": ["construction", "rtdetr_x_fp32", "MiniCPM"], "areas": {"confidence": {"validation_fixed_A25": 0.08994758708890332, "test_fixed_A25": 0.08325267500178912, "test_ranked_A25_diagnostic": 0.08357421552875581, "selected_validation_actual_calls": 1.0, "selected_test_actual_calls": 1.0}, "learned": {"validation_fixed_A25": 0.10056264714432415, "test_fixed_A25": 0.10905069389769234, "test_ranked_A25_diagnostic": 0.1142703643498825, "selected_validation_actual_calls": 0.7728958630527817, "selected_test_actual_calls": 0.7843541944074567}}}, {"key": ["construction", "rtdetr_x_fp32", "Qwen3.8"], "areas": {"confidence": {"validation_fixed_A25": 0.10053453664183931, "test_fixed_A25": 0.08910633276678805, "test_ranked_A25_diagnostic": 0.08907503728424503, "selected_validation_actual_calls": 1.0, "selected_test_actual_calls": 1.0}, "learned": {"validation_fixed_A25": 0.11153717985565076, "test_fixed_A25": 0.11890339192252425, "test_ranked_A25_diagnostic": 0.12154998969679998, "selected_validation_actual_calls": 0.8308131241084166, "selected_test_actual_calls": 0.8356857523302264}}}]
releases/20260913_lr/paper_data/comparisons.json ADDED
The diff for this file is too large to render. See raw diff
 
releases/20260913_lr/paper_data/cost_20260911/construction_router.json ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "component": "router",
3
+ "expert": "yolo26x",
4
+ "task": "construction",
5
+ "endpoint": "Qwen3.8",
6
+ "condition": "both",
7
+ "seed": 42,
8
+ "device": "cuda:0",
9
+ "checkpoint": "/mnt/HithinkOmniSSD/user_workspace/caisihang/research/ICASSP_ModelCollaboration/outputs/router_method_improvement_20260911_r1/conditions/both_yolo26x_Qwen3.8_seed42/best.pt",
10
+ "sample_count": 1024,
11
+ "batch_size": 256,
12
+ "row_count_loaded": 3004,
13
+ "mean_latency_ms_per_sample": 0.004675843752920628,
14
+ "median_latency_ms_per_sample": 0.004634750075638294,
15
+ "latency_ms_per_sample": [
16
+ 0.004889749921858311,
17
+ 0.004717750009149313,
18
+ 0.004544124938547611,
19
+ 0.0045517501421272755
20
+ ],
21
+ "flops_counted_per_batch": 0,
22
+ "gflops_counted_per_sample": 0.0,
23
+ "operators": {},
24
+ "flops_scope": "GPU FlopCounterMode for the router forward; multiply-add counted as 2 FLOPs.",
25
+ "latency_scope": "A800 CUDA forward latency on cached router features, excluding feature loading and model load.",
26
+ "completed": true
27
+ }
releases/20260913_lr/paper_data/cost_20260911/cost_summary.json ADDED
@@ -0,0 +1,1036 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "complete",
3
+ "protocol": {
4
+ "test_results": "C:\\Users\\csh10\\Documents\\CodexProjects\\ICASSP-COLLA\\dse_results\\method_improvement_20260911\\test_results.json",
5
+ "curves_use": "validation-calibrated thresholds transferred unchanged to test",
6
+ "cost_uses": "actual test call fraction",
7
+ "flop_counting": "multiply-add counted as 2 FLOPs; unsupported operations excluded",
8
+ "large_generation_flops": "mean prefill + mean completion tokens * one-token decode probe",
9
+ "seed_aggregation": "mean and population standard deviation across three seeds for learned/original methods"
10
+ },
11
+ "components": {
12
+ "grefcoco|instancevg|MiniCPM": {
13
+ "pair_key": "grefcoco|instancevg|MiniCPM",
14
+ "task": "grefcoco",
15
+ "expert": "instancevg",
16
+ "endpoint": "MiniCPM",
17
+ "label": "gRefCOCO / InstanceVG + MiniCPM",
18
+ "specialist_gflops_per_input": 7.502680064,
19
+ "specialist_forward_latency_ms": 11.246967077255249,
20
+ "specialist_end_to_end_latency_ms": 16.31702448321448,
21
+ "specialist_sample_count": 64,
22
+ "router_gflops_per_input": 0.000954368,
23
+ "router_flop_count": 954368,
24
+ "router_flop_breakdown": {
25
+ "image": 393728,
26
+ "text": 196608,
27
+ "hidden": 196608,
28
+ "confidence": 2048,
29
+ "output": 512,
30
+ "head_input_to_hidden": 163840,
31
+ "head_hidden_to_classes": 1024
32
+ },
33
+ "router_measured_gflops_per_input": 0.0,
34
+ "router_latency_ms": 0.001830718741985038,
35
+ "router_sample_count": 1024,
36
+ "large_prefill_gflops": 7467.756340960001,
37
+ "large_decode_one_token_gflops": 15.445344384,
38
+ "large_mean_completion_tokens": 169.5,
39
+ "large_completion_token_min": 5,
40
+ "large_completion_token_max": 512,
41
+ "large_estimated_full_generation_gflops": 10085.742214048001,
42
+ "large_batch_wall_latency_ms_per_input": 1447.4320754343353,
43
+ "large_measured_sample_count": 30,
44
+ "large_flops_probe_count": 4,
45
+ "flops_formula": "specialist + router + actual_call_fraction * (prefill + mean_completion_tokens * one_token_decode)",
46
+ "latency_formula": "specialist end-to-end + router + actual_call_fraction * large-model batch-wall latency",
47
+ "specialist_flops_scope": "GPU FlopCounterMode dense products, convolutions and supported attention; multiply-add counted as 2 FLOPs. Elementwise, normalization, custom ops and postprocessing may be missing.",
48
+ "specialist_latency_scope": "A800 wall-clock CUDA forward latency and end-to-end latency including image read/preprocess, excluding model load; batch processing uses the recorded batch size.",
49
+ "large_flops_scope": "HF reference execution on the A800 with real task inputs; dense products/convolutions/supported attention counted with multiply-add=2 FLOPs. vLLM custom kernels, elementwise/normalization/nonlinearities and other unsupported operations are not complete.",
50
+ "large_latency_scope": "A800 vLLM-0.18.0 native generation latency with the frozen task prompt and structured-output schema; model loading and warmup excluded. Batch wall time includes the real request/image preparation path."
51
+ },
52
+ "construction|yolo26x|Qwen3.8": {
53
+ "pair_key": "construction|yolo26x|Qwen3.8",
54
+ "task": "construction",
55
+ "expert": "yolo26x",
56
+ "endpoint": "Qwen3.8",
57
+ "label": "Construction / YOLO26x + Qwen3.8",
58
+ "specialist_gflops_per_input": 194.4179712,
59
+ "specialist_forward_latency_ms": 32.08068132400513,
60
+ "specialist_end_to_end_latency_ms": 31.79218670447881,
61
+ "specialist_sample_count": 64,
62
+ "router_gflops_per_input": 0.000726528,
63
+ "router_flop_count": 726528,
64
+ "router_flop_breakdown": {
65
+ "image": 393728,
66
+ "hidden": 196608,
67
+ "confidence": 2048,
68
+ "output": 2048,
69
+ "head_input_to_hidden": 131072,
70
+ "head_hidden_to_classes": 1024
71
+ },
72
+ "router_measured_gflops_per_input": 0.0,
73
+ "router_latency_ms": 0.004675843752920628,
74
+ "router_sample_count": 1024,
75
+ "large_prefill_gflops": 87588.607720992,
76
+ "large_decode_one_token_gflops": 51.905495232,
77
+ "large_mean_completion_tokens": 137.03333333333333,
78
+ "large_completion_token_min": 110,
79
+ "large_completion_token_max": 226,
80
+ "large_estimated_full_generation_gflops": 94701.3907509504,
81
+ "large_batch_wall_latency_ms_per_input": 2599.870914062194,
82
+ "large_measured_sample_count": 30,
83
+ "large_flops_probe_count": 4,
84
+ "flops_formula": "specialist + router + actual_call_fraction * (prefill + mean_completion_tokens * one_token_decode)",
85
+ "latency_formula": "specialist end-to-end + router + actual_call_fraction * large-model batch-wall latency",
86
+ "specialist_flops_scope": "GPU FlopCounterMode around the real Ultralytics predict path; multiply-add counted as 2 FLOPs. Elementwise, postprocessing and unsupported custom ops may be missing.",
87
+ "specialist_latency_scope": "A800 wall-clock latency for real Ultralytics prediction including image loading/preprocess, forward and result postprocessing, excluding model load; batch processing uses the recorded batch size.",
88
+ "large_flops_scope": "HF reference execution on the A800 with real task inputs; dense products/convolutions/supported attention counted with multiply-add=2 FLOPs. vLLM custom kernels, elementwise/normalization/nonlinearities and other unsupported operations are not complete.",
89
+ "large_latency_scope": "A800 vLLM-0.18.0 native generation latency with the frozen task prompt and structured-output schema; model loading and warmup excluded. Batch wall time includes the real request/image preparation path."
90
+ }
91
+ },
92
+ "selected_points": [
93
+ {
94
+ "pair_key": "grefcoco|instancevg|MiniCPM",
95
+ "pair_label": "gRefCOCO / InstanceVG + MiniCPM",
96
+ "task": "grefcoco",
97
+ "method": "both",
98
+ "method_label": "两项同时改",
99
+ "seed_count": 3,
100
+ "selected_nominal_budget_mean": 0.04666666666666667,
101
+ "selected_nominal_budgets": [
102
+ 0.05,
103
+ 0.04,
104
+ 0.05
105
+ ],
106
+ "test_metric": 0.6805338929378291,
107
+ "test_metric_std": 0.0004176374109666599,
108
+ "actual_test_call_fraction": 0.11292289368459858,
109
+ "actual_test_call_fraction_std": 0.012834207132185534,
110
+ "flops_gflops_per_input": 1146.4148301992102,
111
+ "latency_ms_per_input": 179.7670735719058
112
+ },
113
+ {
114
+ "pair_key": "grefcoco|instancevg|MiniCPM",
115
+ "pair_label": "gRefCOCO / InstanceVG + MiniCPM",
116
+ "task": "grefcoco",
117
+ "method": "original",
118
+ "method_label": "原方法",
119
+ "seed_count": 3,
120
+ "selected_nominal_budget_mean": 0.19333333333333333,
121
+ "selected_nominal_budgets": [
122
+ 0.16,
123
+ 0.2,
124
+ 0.22
125
+ ],
126
+ "test_metric": 0.6732741589390202,
127
+ "test_metric_std": 0.0014792314998359972,
128
+ "actual_test_call_fraction": 0.26045240998591535,
129
+ "actual_test_call_fraction_std": 0.019638624816705554,
130
+ "flops_gflops_per_input": 2634.3595005774837,
131
+ "latency_ms_per_input": 393.3060275397443
132
+ },
133
+ {
134
+ "pair_key": "grefcoco|instancevg|MiniCPM",
135
+ "pair_label": "gRefCOCO / InstanceVG + MiniCPM",
136
+ "task": "grefcoco",
137
+ "method": "confidence",
138
+ "method_label": "置信度路由",
139
+ "seed_count": 1,
140
+ "selected_nominal_budget_mean": 0.65,
141
+ "selected_nominal_budgets": [
142
+ 0.65
143
+ ],
144
+ "test_metric": 0.6187788900547316,
145
+ "test_metric_std": 0.0,
146
+ "actual_test_call_fraction": 0.5209426310864078,
147
+ "actual_test_call_fraction_std": 0.0,
148
+ "flops_gflops_per_input": 5261.596719877418,
149
+ "latency_ms_per_input": 770.347928897579
150
+ },
151
+ {
152
+ "pair_key": "construction|yolo26x|Qwen3.8",
153
+ "pair_label": "Construction / YOLO26x + Qwen3.8",
154
+ "task": "construction",
155
+ "method": "both",
156
+ "method_label": "两项同时改",
157
+ "seed_count": 3,
158
+ "selected_nominal_budget_mean": 0.44333333333333336,
159
+ "selected_nominal_budgets": [
160
+ 0.44,
161
+ 0.44,
162
+ 0.45
163
+ ],
164
+ "test_metric": 0.3010820421930382,
165
+ "test_metric_std": 0.001446335574639307,
166
+ "actual_test_call_fraction": 0.4414114513981358,
167
+ "actual_test_call_fraction_std": 0.005785002395799842,
168
+ "flops_gflops_per_input": 41996.69703852701,
169
+ "latency_ms_per_input": 1179.4096561722229
170
+ },
171
+ {
172
+ "pair_key": "construction|yolo26x|Qwen3.8",
173
+ "pair_label": "Construction / YOLO26x + Qwen3.8",
174
+ "task": "construction",
175
+ "method": "original",
176
+ "method_label": "原方法",
177
+ "seed_count": 3,
178
+ "selected_nominal_budget_mean": 0.47333333333333333,
179
+ "selected_nominal_budgets": [
180
+ 0.5,
181
+ 0.46,
182
+ 0.46
183
+ ],
184
+ "test_metric": 0.3000434768989391,
185
+ "test_metric_std": 0.002173546570267329,
186
+ "actual_test_call_fraction": 0.46671105193075896,
187
+ "actual_test_call_fraction_std": 0.01718389127843443,
188
+ "flops_gflops_per_input": 44392.60439440991,
189
+ "latency_ms_per_input": 1245.185351734382
190
+ },
191
+ {
192
+ "pair_key": "construction|yolo26x|Qwen3.8",
193
+ "pair_label": "Construction / YOLO26x + Qwen3.8",
194
+ "task": "construction",
195
+ "method": "confidence",
196
+ "method_label": "置信度路由",
197
+ "seed_count": 1,
198
+ "selected_nominal_budget_mean": 0.93,
199
+ "selected_nominal_budgets": [
200
+ 0.93
201
+ ],
202
+ "test_metric": 0.2597490004762105,
203
+ "test_metric_std": 0.0,
204
+ "actual_test_call_fraction": 0.9221038615179761,
205
+ "actual_test_call_fraction_std": 0.0,
206
+ "flops_gflops_per_input": 87518.93680030212,
207
+ "latency_ms_per_input": 2429.147871853251
208
+ }
209
+ ],
210
+ "fixed_budget_points": [
211
+ {
212
+ "pair_key": "grefcoco|instancevg|MiniCPM",
213
+ "pair_label": "gRefCOCO / InstanceVG + MiniCPM",
214
+ "task": "grefcoco",
215
+ "method": "both",
216
+ "method_label": "两项同时改",
217
+ "nominal_budget": 0.0,
218
+ "seed_count": 3,
219
+ "test_metric": 0.6695119530385957,
220
+ "test_metric_std": 0.0,
221
+ "actual_test_call_fraction": 0.0,
222
+ "actual_test_call_fraction_std": 0.0,
223
+ "flops_gflops_per_input": 7.503634432,
224
+ "latency_ms_per_input": 16.318855201956467
225
+ },
226
+ {
227
+ "pair_key": "grefcoco|instancevg|MiniCPM",
228
+ "pair_label": "gRefCOCO / InstanceVG + MiniCPM",
229
+ "task": "grefcoco",
230
+ "method": "both",
231
+ "method_label": "两项同时改",
232
+ "nominal_budget": 0.01,
233
+ "seed_count": 3,
234
+ "test_metric": 0.6743328701471798,
235
+ "test_metric_std": 0.0002504540415259137,
236
+ "actual_test_call_fraction": 0.021268751949635595,
237
+ "actual_test_call_fraction_std": 0.0024468276041848553,
238
+ "flops_gflops_per_input": 222.01478381055546,
239
+ "latency_ms_per_input": 47.10392897831558
240
+ },
241
+ {
242
+ "pair_key": "grefcoco|instancevg|MiniCPM",
243
+ "pair_label": "gRefCOCO / InstanceVG + MiniCPM",
244
+ "task": "grefcoco",
245
+ "method": "both",
246
+ "method_label": "两项同时改",
247
+ "nominal_budget": 0.05,
248
+ "seed_count": 3,
249
+ "test_metric": 0.6805811568310505,
250
+ "test_metric_std": 0.0004271568320976906,
251
+ "actual_test_call_fraction": 0.11976670542305912,
252
+ "actual_test_call_fraction_std": 0.0032185843726325893,
253
+ "flops_gflops_per_input": 1215.4397511547988,
254
+ "latency_ms_per_input": 189.67302620038757
255
+ },
256
+ {
257
+ "pair_key": "grefcoco|instancevg|MiniCPM",
258
+ "pair_label": "gRefCOCO / InstanceVG + MiniCPM",
259
+ "task": "grefcoco",
260
+ "method": "both",
261
+ "method_label": "两项同时改",
262
+ "nominal_budget": 0.1,
263
+ "seed_count": 3,
264
+ "test_metric": 0.6804866290446077,
265
+ "test_metric_std": 0.0005385595679416328,
266
+ "actual_test_call_fraction": 0.172031118547297,
267
+ "actual_test_call_fraction_std": 0.0033259813564282133,
268
+ "flops_gflops_per_input": 1742.5651488943693,
269
+ "latency_ms_per_input": 265.32221416016074
270
+ },
271
+ {
272
+ "pair_key": "grefcoco|instancevg|MiniCPM",
273
+ "pair_label": "gRefCOCO / InstanceVG + MiniCPM",
274
+ "task": "grefcoco",
275
+ "method": "both",
276
+ "method_label": "两项同时改",
277
+ "nominal_budget": 0.25,
278
+ "seed_count": 3,
279
+ "test_metric": 0.6794184650578038,
280
+ "test_metric_std": 0.00038210745789084077,
281
+ "actual_test_call_fraction": 0.24530905859777485,
282
+ "actual_test_call_fraction_std": 0.009454413833556307,
283
+ "flops_gflops_per_input": 2481.627562219953,
284
+ "latency_ms_per_input": 371.38705501097667
285
+ },
286
+ {
287
+ "pair_key": "grefcoco|instancevg|MiniCPM",
288
+ "pair_label": "gRefCOCO / InstanceVG + MiniCPM",
289
+ "task": "grefcoco",
290
+ "method": "both",
291
+ "method_label": "两项同时改",
292
+ "nominal_budget": 0.5,
293
+ "seed_count": 3,
294
+ "test_metric": 0.664568149807636,
295
+ "test_metric_std": 0.0016062493473305785,
296
+ "actual_test_call_fraction": 0.4310372534006371,
297
+ "actual_test_call_fraction_std": 0.0088737075287505,
298
+ "flops_gflops_per_input": 4354.834256882111,
299
+ "latency_ms_per_input": 640.2160014811561
300
+ },
301
+ {
302
+ "pair_key": "grefcoco|instancevg|MiniCPM",
303
+ "pair_label": "gRefCOCO / InstanceVG + MiniCPM",
304
+ "task": "grefcoco",
305
+ "method": "both",
306
+ "method_label": "两项同时改",
307
+ "nominal_budget": 0.75,
308
+ "seed_count": 3,
309
+ "test_metric": 0.6150355897115957,
310
+ "test_metric_std": 0.0013848240058531283,
311
+ "actual_test_call_fraction": 0.7057350008034863,
312
+ "actual_test_call_fraction_std": 0.006014701362190238,
313
+ "flops_gflops_per_input": 7125.364923966921,
314
+ "latency_ms_per_input": 1037.8223321215987
315
+ },
316
+ {
317
+ "pair_key": "grefcoco|instancevg|MiniCPM",
318
+ "pair_label": "gRefCOCO / InstanceVG + MiniCPM",
319
+ "task": "grefcoco",
320
+ "method": "both",
321
+ "method_label": "两项同时改",
322
+ "nominal_budget": 0.99,
323
+ "seed_count": 3,
324
+ "test_metric": 0.5069147075782926,
325
+ "test_metric_std": 9.357773432602366e-05,
326
+ "actual_test_call_fraction": 0.988051687793627,
327
+ "actual_test_call_fraction_std": 0.00048144255369636903,
328
+ "flops_gflops_per_input": 9972.73825167356,
329
+ "latency_ms_per_input": 1446.4565603014837
330
+ },
331
+ {
332
+ "pair_key": "grefcoco|instancevg|MiniCPM",
333
+ "pair_label": "gRefCOCO / InstanceVG + MiniCPM",
334
+ "task": "grefcoco",
335
+ "method": "both",
336
+ "method_label": "两项同时改",
337
+ "nominal_budget": 1.0,
338
+ "seed_count": 3,
339
+ "test_metric": 0.5000708958398321,
340
+ "test_metric_std": 0.0,
341
+ "actual_test_call_fraction": 1.0,
342
+ "actual_test_call_fraction_std": 0.0,
343
+ "flops_gflops_per_input": 10093.24584848,
344
+ "latency_ms_per_input": 1463.7509306362917
345
+ },
346
+ {
347
+ "pair_key": "grefcoco|instancevg|MiniCPM",
348
+ "pair_label": "gRefCOCO / InstanceVG + MiniCPM",
349
+ "task": "grefcoco",
350
+ "method": "original",
351
+ "method_label": "原方法",
352
+ "nominal_budget": 0.0,
353
+ "seed_count": 3,
354
+ "test_metric": 0.6695119530385957,
355
+ "test_metric_std": 0.0,
356
+ "actual_test_call_fraction": 0.0,
357
+ "actual_test_call_fraction_std": 0.0,
358
+ "flops_gflops_per_input": 7.503634432,
359
+ "latency_ms_per_input": 16.318855201956467
360
+ },
361
+ {
362
+ "pair_key": "grefcoco|instancevg|MiniCPM",
363
+ "pair_label": "gRefCOCO / InstanceVG + MiniCPM",
364
+ "task": "grefcoco",
365
+ "method": "original",
366
+ "method_label": "原方法",
367
+ "nominal_budget": 0.01,
368
+ "seed_count": 3,
369
+ "test_metric": 0.6742194368034484,
370
+ "test_metric_std": 0.0005415378002401663,
371
+ "actual_test_call_fraction": 0.04492905689627466,
372
+ "actual_test_call_fraction_std": 0.0027758676387843045,
373
+ "flops_gflops_per_input": 460.64652020812184,
374
+ "latency_ms_per_input": 81.35061327263864
375
+ },
376
+ {
377
+ "pair_key": "grefcoco|instancevg|MiniCPM",
378
+ "pair_label": "gRefCOCO / InstanceVG + MiniCPM",
379
+ "task": "grefcoco",
380
+ "method": "original",
381
+ "method_label": "原方法",
382
+ "nominal_budget": 0.05,
383
+ "seed_count": 3,
384
+ "test_metric": 0.679891103990018,
385
+ "test_metric_std": 0.0007177890142290943,
386
+ "actual_test_call_fraction": 0.10489748461560276,
387
+ "actual_test_call_fraction_std": 0.007840998690258661,
388
+ "flops_gflops_per_input": 1065.4726231670354,
389
+ "latency_ms_per_input": 168.15083906695963
390
+ },
391
+ {
392
+ "pair_key": "grefcoco|instancevg|MiniCPM",
393
+ "pair_label": "gRefCOCO / InstanceVG + MiniCPM",
394
+ "task": "grefcoco",
395
+ "method": "original",
396
+ "method_label": "原方法",
397
+ "nominal_budget": 0.1,
398
+ "seed_count": 3,
399
+ "test_metric": 0.680127423456125,
400
+ "test_metric_std": 0.0007907632127485806,
401
+ "actual_test_call_fraction": 0.16706840975904869,
402
+ "actual_test_call_fraction_std": 0.004046760924054268,
403
+ "flops_gflops_per_input": 1692.5125473727062,
404
+ "latency_ms_per_input": 258.1390302790103
405
+ },
406
+ {
407
+ "pair_key": "grefcoco|instancevg|MiniCPM",
408
+ "pair_label": "gRefCOCO / InstanceVG + MiniCPM",
409
+ "task": "grefcoco",
410
+ "method": "original",
411
+ "method_label": "原方法",
412
+ "nominal_budget": 0.25,
413
+ "seed_count": 3,
414
+ "test_metric": 0.6658726332605469,
415
+ "test_metric_std": 0.0048175982262131,
416
+ "actual_test_call_fraction": 0.3117620924670807,
417
+ "actual_test_call_fraction_std": 0.009206880232216604,
418
+ "flops_gflops_per_input": 3151.855731167172,
419
+ "latency_ms_per_input": 467.5733077433342
420
+ },
421
+ {
422
+ "pair_key": "grefcoco|instancevg|MiniCPM",
423
+ "pair_label": "gRefCOCO / InstanceVG + MiniCPM",
424
+ "task": "grefcoco",
425
+ "method": "original",
426
+ "method_label": "原方法",
427
+ "nominal_budget": 0.5,
428
+ "seed_count": 3,
429
+ "test_metric": 0.6273242019491629,
430
+ "test_metric_std": 0.009832598413097358,
431
+ "actual_test_call_fraction": 0.5310665570144345,
432
+ "actual_test_call_fraction_std": 0.009248673211785196,
433
+ "flops_gflops_per_input": 5363.704026981612,
434
+ "latency_ms_per_input": 785.0016240151261
435
+ },
436
+ {
437
+ "pair_key": "grefcoco|instancevg|MiniCPM",
438
+ "pair_label": "gRefCOCO / InstanceVG + MiniCPM",
439
+ "task": "grefcoco",
440
+ "method": "original",
441
+ "method_label": "原方法",
442
+ "nominal_budget": 0.75,
443
+ "seed_count": 3,
444
+ "test_metric": 0.5814404144098159,
445
+ "test_metric_std": 0.00858033252822142,
446
+ "actual_test_call_fraction": 0.7394719677849304,
447
+ "actual_test_call_fraction_std": 0.008502993761525655,
448
+ "flops_gflops_per_input": 7465.627276025616,
449
+ "latency_ms_per_input": 1086.65430025841
450
+ },
451
+ {
452
+ "pair_key": "grefcoco|instancevg|MiniCPM",
453
+ "pair_label": "gRefCOCO / InstanceVG + MiniCPM",
454
+ "task": "grefcoco",
455
+ "method": "original",
456
+ "method_label": "原方法",
457
+ "nominal_budget": 0.99,
458
+ "seed_count": 3,
459
+ "test_metric": 0.5100152189736173,
460
+ "test_metric_std": 0.0036056620177345906,
461
+ "actual_test_call_fraction": 0.978079006323909,
462
+ "actual_test_call_fraction_std": 0.009338895613008017,
463
+ "flops_gflops_per_input": 9872.15635718717,
464
+ "latency_ms_per_input": 1432.0217812641245
465
+ },
466
+ {
467
+ "pair_key": "grefcoco|instancevg|MiniCPM",
468
+ "pair_label": "gRefCOCO / InstanceVG + MiniCPM",
469
+ "task": "grefcoco",
470
+ "method": "original",
471
+ "method_label": "原方法",
472
+ "nominal_budget": 1.0,
473
+ "seed_count": 3,
474
+ "test_metric": 0.5000708958398321,
475
+ "test_metric_std": 0.0,
476
+ "actual_test_call_fraction": 1.0,
477
+ "actual_test_call_fraction_std": 0.0,
478
+ "flops_gflops_per_input": 10093.24584848,
479
+ "latency_ms_per_input": 1463.7509306362917
480
+ },
481
+ {
482
+ "pair_key": "grefcoco|instancevg|MiniCPM",
483
+ "pair_label": "gRefCOCO / InstanceVG + MiniCPM",
484
+ "task": "grefcoco",
485
+ "method": "confidence",
486
+ "method_label": "置信度路由",
487
+ "nominal_budget": 0.0,
488
+ "seed_count": 1,
489
+ "test_metric": 0.6695119530385957,
490
+ "test_metric_std": 0.0,
491
+ "actual_test_call_fraction": 0.0,
492
+ "actual_test_call_fraction_std": 0.0,
493
+ "flops_gflops_per_input": 7.503634432,
494
+ "latency_ms_per_input": 16.318855201956467
495
+ },
496
+ {
497
+ "pair_key": "grefcoco|instancevg|MiniCPM",
498
+ "pair_label": "gRefCOCO / InstanceVG + MiniCPM",
499
+ "task": "grefcoco",
500
+ "method": "confidence",
501
+ "method_label": "置信度路由",
502
+ "nominal_budget": 0.01,
503
+ "seed_count": 1,
504
+ "test_metric": 0.6686044862887446,
505
+ "test_metric_std": 0.0,
506
+ "actual_test_call_fraction": 0.002467175226157729,
507
+ "actual_test_call_fraction_std": 0.0,
508
+ "flops_gflops_per_input": 32.38692775991243,
509
+ "latency_ms_per_input": 19.889923760014124
510
+ },
511
+ {
512
+ "pair_key": "grefcoco|instancevg|MiniCPM",
513
+ "pair_label": "gRefCOCO / InstanceVG + MiniCPM",
514
+ "task": "grefcoco",
515
+ "method": "confidence",
516
+ "method_label": "置信度路由",
517
+ "nominal_budget": 0.05,
518
+ "seed_count": 1,
519
+ "test_metric": 0.6662791027422511,
520
+ "test_metric_std": 0.0,
521
+ "actual_test_call_fraction": 0.012534384482318578,
522
+ "actual_test_call_fraction_std": 0.0,
523
+ "flops_gflops_per_input": 133.9222051324287,
524
+ "latency_ms_per_input": 34.46152534749078
525
+ },
526
+ {
527
+ "pair_key": "grefcoco|instancevg|MiniCPM",
528
+ "pair_label": "gRefCOCO / InstanceVG + MiniCPM",
529
+ "task": "grefcoco",
530
+ "method": "confidence",
531
+ "method_label": "置信度路由",
532
+ "nominal_budget": 0.1,
533
+ "seed_count": 1,
534
+ "test_metric": 0.6645208859144145,
535
+ "test_metric_std": 0.0,
536
+ "actual_test_call_fraction": 0.0345971698380739,
537
+ "actual_test_call_fraction_std": 0.0,
538
+ "flops_gflops_per_input": 356.4417707544502,
539
+ "latency_ms_per_input": 66.39590854483396
540
+ },
541
+ {
542
+ "pair_key": "grefcoco|instancevg|MiniCPM",
543
+ "pair_label": "gRefCOCO / InstanceVG + MiniCPM",
544
+ "task": "grefcoco",
545
+ "method": "confidence",
546
+ "method_label": "置信度路由",
547
+ "nominal_budget": 0.25,
548
+ "seed_count": 1,
549
+ "test_metric": 0.6596999688058305,
550
+ "test_metric_std": 0.0,
551
+ "actual_test_call_fraction": 0.12477667810452883,
552
+ "actual_test_call_fraction_std": 0.0,
553
+ "flops_gflops_per_input": 1265.9690441195253,
554
+ "latency_ms_per_input": 196.92462135659662
555
+ },
556
+ {
557
+ "pair_key": "grefcoco|instancevg|MiniCPM",
558
+ "pair_label": "gRefCOCO / InstanceVG + MiniCPM",
559
+ "task": "grefcoco",
560
+ "method": "confidence",
561
+ "method_label": "置信度路由",
562
+ "nominal_budget": 0.5,
563
+ "seed_count": 1,
564
+ "test_metric": 0.6527521765022829,
565
+ "test_metric_std": 0.0,
566
+ "actual_test_call_fraction": 0.2345234381646485,
567
+ "actual_test_call_fraction_std": 0.0,
568
+ "flops_gflops_per_input": 2372.8465749128713,
569
+ "latency_ms_per_input": 355.77560204260965
570
+ },
571
+ {
572
+ "pair_key": "grefcoco|instancevg|MiniCPM",
573
+ "pair_label": "gRefCOCO / InstanceVG + MiniCPM",
574
+ "task": "grefcoco",
575
+ "method": "confidence",
576
+ "method_label": "置信度路由",
577
+ "nominal_budget": 0.75,
578
+ "seed_count": 1,
579
+ "test_metric": 0.567762243711539,
580
+ "test_metric_std": 0.0,
581
+ "actual_test_call_fraction": 0.7478093185491875,
582
+ "actual_test_call_fraction_std": 0.0,
583
+ "flops_gflops_per_input": 7549.715646582009,
584
+ "latency_ms_per_input": 1098.722049178743
585
+ },
586
+ {
587
+ "pair_key": "grefcoco|instancevg|MiniCPM",
588
+ "pair_label": "gRefCOCO / InstanceVG + MiniCPM",
589
+ "task": "grefcoco",
590
+ "method": "confidence",
591
+ "method_label": "置信度路由",
592
+ "nominal_budget": 0.99,
593
+ "seed_count": 1,
594
+ "test_metric": 0.5019425460114001,
595
+ "test_metric_std": 0.0,
596
+ "actual_test_call_fraction": 0.9908118991577575,
597
+ "actual_test_call_fraction_std": 0.0,
598
+ "flops_gflops_per_input": 10000.577031948465,
599
+ "latency_ms_per_input": 1450.4517787649047
600
+ },
601
+ {
602
+ "pair_key": "grefcoco|instancevg|MiniCPM",
603
+ "pair_label": "gRefCOCO / InstanceVG + MiniCPM",
604
+ "task": "grefcoco",
605
+ "method": "confidence",
606
+ "method_label": "置信度路由",
607
+ "nominal_budget": 1.0,
608
+ "seed_count": 1,
609
+ "test_metric": 0.5000708958398321,
610
+ "test_metric_std": 0.0,
611
+ "actual_test_call_fraction": 1.0,
612
+ "actual_test_call_fraction_std": 0.0,
613
+ "flops_gflops_per_input": 10093.24584848,
614
+ "latency_ms_per_input": 1463.7509306362917
615
+ },
616
+ {
617
+ "pair_key": "construction|yolo26x|Qwen3.8",
618
+ "pair_label": "Construction / YOLO26x + Qwen3.8",
619
+ "task": "construction",
620
+ "method": "both",
621
+ "method_label": "两项同时改",
622
+ "nominal_budget": 0.0,
623
+ "seed_count": 3,
624
+ "test_metric": 0.18129624718318374,
625
+ "test_metric_std": 2.7755575615628914e-17,
626
+ "actual_test_call_fraction": 0.0,
627
+ "actual_test_call_fraction_std": 0.0,
628
+ "flops_gflops_per_input": 194.418697728,
629
+ "latency_ms_per_input": 31.79686254823173
630
+ },
631
+ {
632
+ "pair_key": "construction|yolo26x|Qwen3.8",
633
+ "pair_label": "Construction / YOLO26x + Qwen3.8",
634
+ "task": "construction",
635
+ "method": "both",
636
+ "method_label": "两项同时改",
637
+ "nominal_budget": 0.01,
638
+ "seed_count": 3,
639
+ "test_metric": 0.1834834493716783,
640
+ "test_metric_std": 0.0012311365629078506,
641
+ "actual_test_call_fraction": 0.01187305814469596,
642
+ "actual_test_call_fraction_std": 0.0025254786233335643,
643
+ "flops_gflops_per_input": 1318.8138164976065,
644
+ "latency_ms_per_input": 62.665281079596
645
+ },
646
+ {
647
+ "pair_key": "construction|yolo26x|Qwen3.8",
648
+ "pair_label": "Construction / YOLO26x + Qwen3.8",
649
+ "task": "construction",
650
+ "method": "both",
651
+ "method_label": "两项同时改",
652
+ "nominal_budget": 0.05,
653
+ "seed_count": 3,
654
+ "test_metric": 0.19244312207391248,
655
+ "test_metric_std": 0.0014149660525805208,
656
+ "actual_test_call_fraction": 0.04804704837993786,
657
+ "actual_test_call_fraction_std": 0.0025254786233335656,
658
+ "flops_gflops_per_input": 4744.541000786314,
659
+ "latency_ms_per_input": 156.71298613777122
660
+ },
661
+ {
662
+ "pair_key": "construction|yolo26x|Qwen3.8",
663
+ "pair_label": "Construction / YOLO26x + Qwen3.8",
664
+ "task": "construction",
665
+ "method": "both",
666
+ "method_label": "两项同时改",
667
+ "nominal_budget": 0.1,
668
+ "seed_count": 3,
669
+ "test_metric": 0.2058996264309502,
670
+ "test_metric_std": 0.000756529123691676,
671
+ "actual_test_call_fraction": 0.10707944962272525,
672
+ "actual_test_call_fraction_std": 0.0037002512207130658,
673
+ "flops_gflops_per_input": 10334.991497846413,
674
+ "latency_ms_per_input": 310.1896091161431
675
+ },
676
+ {
677
+ "pair_key": "construction|yolo26x|Qwen3.8",
678
+ "pair_label": "Construction / YOLO26x + Qwen3.8",
679
+ "task": "construction",
680
+ "method": "both",
681
+ "method_label": "两项同时改",
682
+ "nominal_budget": 0.25,
683
+ "seed_count": 3,
684
+ "test_metric": 0.25617136079673564,
685
+ "test_metric_std": 0.0020077364088740926,
686
+ "actual_test_call_fraction": 0.26498002663115844,
687
+ "actual_test_call_fraction_std": 0.008293336414138841,
688
+ "flops_gflops_per_input": 25288.39574092258,
689
+ "latency_ms_per_input": 720.7107265940061
690
+ },
691
+ {
692
+ "pair_key": "construction|yolo26x|Qwen3.8",
693
+ "pair_label": "Construction / YOLO26x + Qwen3.8",
694
+ "task": "construction",
695
+ "method": "both",
696
+ "method_label": "两项同时改",
697
+ "nominal_budget": 0.5,
698
+ "seed_count": 3,
699
+ "test_metric": 0.29980106360416753,
700
+ "test_metric_std": 0.002221016752728669,
701
+ "actual_test_call_fraction": 0.49511762094984463,
702
+ "actual_test_call_fraction_std": 0.00440231567351916,
703
+ "flops_gflops_per_input": 47082.74598698018,
704
+ "latency_ms_per_input": 1319.038764295403
705
+ },
706
+ {
707
+ "pair_key": "construction|yolo26x|Qwen3.8",
708
+ "pair_label": "Construction / YOLO26x + Qwen3.8",
709
+ "task": "construction",
710
+ "method": "both",
711
+ "method_label": "两项同时改",
712
+ "nominal_budget": 0.75,
713
+ "seed_count": 3,
714
+ "test_metric": 0.28788636673929996,
715
+ "test_metric_std": 0.0011284209510194398,
716
+ "actual_test_call_fraction": 0.7496671105193076,
717
+ "actual_test_call_fraction_std": 0.005429263125932131,
718
+ "flops_gflops_per_input": 71188.93666415288,
719
+ "latency_ms_per_input": 1980.8345784164278
720
+ },
721
+ {
722
+ "pair_key": "construction|yolo26x|Qwen3.8",
723
+ "pair_label": "Construction / YOLO26x + Qwen3.8",
724
+ "task": "construction",
725
+ "method": "both",
726
+ "method_label": "两项同时改",
727
+ "nominal_budget": 0.99,
728
+ "seed_count": 3,
729
+ "test_metric": 0.27239176103160984,
730
+ "test_metric_std": 0.0005578690368974412,
731
+ "actual_test_call_fraction": 0.9862405681313805,
732
+ "actual_test_call_fraction_std": 0.0023116585281845767,
733
+ "flops_gflops_per_input": 93592.7721147772,
734
+ "latency_ms_per_input": 2595.8950299011817
735
+ },
736
+ {
737
+ "pair_key": "construction|yolo26x|Qwen3.8",
738
+ "pair_label": "Construction / YOLO26x + Qwen3.8",
739
+ "task": "construction",
740
+ "method": "both",
741
+ "method_label": "两项同时改",
742
+ "nominal_budget": 1.0,
743
+ "seed_count": 3,
744
+ "test_metric": 0.27001009067886245,
745
+ "test_metric_std": 0.0,
746
+ "actual_test_call_fraction": 1.0,
747
+ "actual_test_call_fraction_std": 0.0,
748
+ "flops_gflops_per_input": 94895.80944867841,
749
+ "latency_ms_per_input": 2631.667776610426
750
+ },
751
+ {
752
+ "pair_key": "construction|yolo26x|Qwen3.8",
753
+ "pair_label": "Construction / YOLO26x + Qwen3.8",
754
+ "task": "construction",
755
+ "method": "original",
756
+ "method_label": "原方法",
757
+ "nominal_budget": 0.0,
758
+ "seed_count": 3,
759
+ "test_metric": 0.18129624718318374,
760
+ "test_metric_std": 2.7755575615628914e-17,
761
+ "actual_test_call_fraction": 0.0,
762
+ "actual_test_call_fraction_std": 0.0,
763
+ "flops_gflops_per_input": 194.418697728,
764
+ "latency_ms_per_input": 31.79686254823173
765
+ },
766
+ {
767
+ "pair_key": "construction|yolo26x|Qwen3.8",
768
+ "pair_label": "Construction / YOLO26x + Qwen3.8",
769
+ "task": "construction",
770
+ "method": "original",
771
+ "method_label": "原方法",
772
+ "nominal_budget": 0.01,
773
+ "seed_count": 3,
774
+ "test_metric": 0.18300903853620068,
775
+ "test_metric_std": 0.000992883110506064,
776
+ "actual_test_call_fraction": 0.010319573901464712,
777
+ "actual_test_call_fraction_std": 0.0032954377285658006,
778
+ "flops_gflops_per_input": 1171.6966981539194,
779
+ "latency_ms_per_input": 58.626422580165155
780
+ },
781
+ {
782
+ "pair_key": "construction|yolo26x|Qwen3.8",
783
+ "pair_label": "Construction / YOLO26x + Qwen3.8",
784
+ "task": "construction",
785
+ "method": "original",
786
+ "method_label": "原方法",
787
+ "nominal_budget": 0.05,
788
+ "seed_count": 3,
789
+ "test_metric": 0.19493527207894204,
790
+ "test_metric_std": 0.0032238119689520237,
791
+ "actual_test_call_fraction": 0.049822458943630715,
792
+ "actual_test_call_fraction_std": 0.011397380483895993,
793
+ "flops_gflops_per_input": 4912.674850321956,
794
+ "latency_ms_per_input": 161.32882442283505
795
+ },
796
+ {
797
+ "pair_key": "construction|yolo26x|Qwen3.8",
798
+ "pair_label": "Construction / YOLO26x + Qwen3.8",
799
+ "task": "construction",
800
+ "method": "original",
801
+ "method_label": "原方法",
802
+ "nominal_budget": 0.1,
803
+ "seed_count": 3,
804
+ "test_metric": 0.20346296196880112,
805
+ "test_metric_std": 0.004110692938093332,
806
+ "actual_test_call_fraction": 0.09576120727918332,
807
+ "actual_test_call_fraction_std": 0.006259345218007303,
808
+ "flops_gflops_per_input": 9263.138207056696,
809
+ "latency_ms_per_input": 280.76364004886125
810
+ },
811
+ {
812
+ "pair_key": "construction|yolo26x|Qwen3.8",
813
+ "pair_label": "Construction / YOLO26x + Qwen3.8",
814
+ "task": "construction",
815
+ "method": "original",
816
+ "method_label": "原方法",
817
+ "nominal_budget": 0.25,
818
+ "seed_count": 3,
819
+ "test_metric": 0.24461408052105227,
820
+ "test_metric_std": 0.0022654085644492608,
821
+ "actual_test_call_fraction": 0.2533288948069241,
822
+ "actual_test_call_fraction_std": 0.0017823322685540256,
823
+ "flops_gflops_per_input": 24185.01735334493,
824
+ "latency_ms_per_input": 690.419287848275
825
+ },
826
+ {
827
+ "pair_key": "construction|yolo26x|Qwen3.8",
828
+ "pair_label": "Construction / YOLO26x + Qwen3.8",
829
+ "task": "construction",
830
+ "method": "original",
831
+ "method_label": "原方法",
832
+ "nominal_budget": 0.5,
833
+ "seed_count": 3,
834
+ "test_metric": 0.29849806777867033,
835
+ "test_metric_std": 0.0015269090065487179,
836
+ "actual_test_call_fraction": 0.4957833999112295,
837
+ "actual_test_call_fraction_std": 0.00518804692745715,
838
+ "flops_gflops_per_input": 47145.796180556055,
839
+ "latency_ms_per_input": 1320.7697036523023
840
+ },
841
+ {
842
+ "pair_key": "construction|yolo26x|Qwen3.8",
843
+ "pair_label": "Construction / YOLO26x + Qwen3.8",
844
+ "task": "construction",
845
+ "method": "original",
846
+ "method_label": "原方法",
847
+ "nominal_budget": 0.75,
848
+ "seed_count": 3,
849
+ "test_metric": 0.28532383356176916,
850
+ "test_metric_std": 0.0029913445567370696,
851
+ "actual_test_call_fraction": 0.7654238792720817,
852
+ "actual_test_call_fraction_std": 0.006553405708048556,
853
+ "flops_gflops_per_input": 72681.1245787817,
854
+ "latency_ms_per_input": 2021.8001431963692
855
+ },
856
+ {
857
+ "pair_key": "construction|yolo26x|Qwen3.8",
858
+ "pair_label": "Construction / YOLO26x + Qwen3.8",
859
+ "task": "construction",
860
+ "method": "original",
861
+ "method_label": "原方法",
862
+ "nominal_budget": 0.99,
863
+ "seed_count": 3,
864
+ "test_metric": 0.27053969734098926,
865
+ "test_metric_std": 0.0014164533171887024,
866
+ "actual_test_call_fraction": 0.9897913892587661,
867
+ "actual_test_call_fraction_std": 0.0027895705925380728,
868
+ "flops_gflops_per_input": 93929.03981384847,
869
+ "latency_ms_per_input": 2605.126706471309
870
+ },
871
+ {
872
+ "pair_key": "construction|yolo26x|Qwen3.8",
873
+ "pair_label": "Construction / YOLO26x + Qwen3.8",
874
+ "task": "construction",
875
+ "method": "original",
876
+ "method_label": "原方法",
877
+ "nominal_budget": 1.0,
878
+ "seed_count": 3,
879
+ "test_metric": 0.27001009067886245,
880
+ "test_metric_std": 0.0,
881
+ "actual_test_call_fraction": 1.0,
882
+ "actual_test_call_fraction_std": 0.0,
883
+ "flops_gflops_per_input": 94895.80944867841,
884
+ "latency_ms_per_input": 2631.667776610426
885
+ },
886
+ {
887
+ "pair_key": "construction|yolo26x|Qwen3.8",
888
+ "pair_label": "Construction / YOLO26x + Qwen3.8",
889
+ "task": "construction",
890
+ "method": "confidence",
891
+ "method_label": "置信度路由",
892
+ "nominal_budget": 0.0,
893
+ "seed_count": 1,
894
+ "test_metric": 0.18129624718318377,
895
+ "test_metric_std": 0.0,
896
+ "actual_test_call_fraction": 0.0,
897
+ "actual_test_call_fraction_std": 0.0,
898
+ "flops_gflops_per_input": 194.418697728,
899
+ "latency_ms_per_input": 31.79686254823173
900
+ },
901
+ {
902
+ "pair_key": "construction|yolo26x|Qwen3.8",
903
+ "pair_label": "Construction / YOLO26x + Qwen3.8",
904
+ "task": "construction",
905
+ "method": "confidence",
906
+ "method_label": "置信度路由",
907
+ "nominal_budget": 0.01,
908
+ "seed_count": 1,
909
+ "test_metric": 0.18129624718318377,
910
+ "test_metric_std": 0.0,
911
+ "actual_test_call_fraction": 0.0,
912
+ "actual_test_call_fraction_std": 0.0,
913
+ "flops_gflops_per_input": 194.418697728,
914
+ "latency_ms_per_input": 31.79686254823173
915
+ },
916
+ {
917
+ "pair_key": "construction|yolo26x|Qwen3.8",
918
+ "pair_label": "Construction / YOLO26x + Qwen3.8",
919
+ "task": "construction",
920
+ "method": "confidence",
921
+ "method_label": "置信度路由",
922
+ "nominal_budget": 0.05,
923
+ "seed_count": 1,
924
+ "test_metric": 0.18129624718318377,
925
+ "test_metric_std": 0.0,
926
+ "actual_test_call_fraction": 0.0,
927
+ "actual_test_call_fraction_std": 0.0,
928
+ "flops_gflops_per_input": 194.418697728,
929
+ "latency_ms_per_input": 31.79686254823173
930
+ },
931
+ {
932
+ "pair_key": "construction|yolo26x|Qwen3.8",
933
+ "pair_label": "Construction / YOLO26x + Qwen3.8",
934
+ "task": "construction",
935
+ "method": "confidence",
936
+ "method_label": "置信度路由",
937
+ "nominal_budget": 0.1,
938
+ "seed_count": 1,
939
+ "test_metric": 0.18129624718318377,
940
+ "test_metric_std": 0.0,
941
+ "actual_test_call_fraction": 0.0,
942
+ "actual_test_call_fraction_std": 0.0,
943
+ "flops_gflops_per_input": 194.418697728,
944
+ "latency_ms_per_input": 31.79686254823173
945
+ },
946
+ {
947
+ "pair_key": "construction|yolo26x|Qwen3.8",
948
+ "pair_label": "Construction / YOLO26x + Qwen3.8",
949
+ "task": "construction",
950
+ "method": "confidence",
951
+ "method_label": "置信度路由",
952
+ "nominal_budget": 0.25,
953
+ "seed_count": 1,
954
+ "test_metric": 0.18026802885144627,
955
+ "test_metric_std": 0.0,
956
+ "actual_test_call_fraction": 0.24800266311584554,
957
+ "actual_test_call_fraction_std": 0.0,
958
+ "flops_gflops_per_input": 23680.615804738005,
959
+ "latency_ms_per_input": 676.5717729930834
960
+ },
961
+ {
962
+ "pair_key": "construction|yolo26x|Qwen3.8",
963
+ "pair_label": "Construction / YOLO26x + Qwen3.8",
964
+ "task": "construction",
965
+ "method": "confidence",
966
+ "method_label": "置信度路由",
967
+ "nominal_budget": 0.5,
968
+ "seed_count": 1,
969
+ "test_metric": 0.18015785955296285,
970
+ "test_metric_std": 0.0,
971
+ "actual_test_call_fraction": 0.5013315579227696,
972
+ "actual_test_call_fraction_std": 0.0,
973
+ "flops_gflops_per_input": 47671.21446035493,
974
+ "latency_ms_per_input": 1335.1941982931266
975
+ },
976
+ {
977
+ "pair_key": "construction|yolo26x|Qwen3.8",
978
+ "pair_label": "Construction / YOLO26x + Qwen3.8",
979
+ "task": "construction",
980
+ "method": "confidence",
981
+ "method_label": "置信度路由",
982
+ "nominal_budget": 0.75,
983
+ "seed_count": 1,
984
+ "test_metric": 0.22103445955201576,
985
+ "test_metric_std": 0.0,
986
+ "actual_test_call_fraction": 0.7486684420772304,
987
+ "actual_test_call_fraction_std": 0.0,
988
+ "flops_gflops_per_input": 71094.36137378908,
989
+ "latency_ms_per_input": 1978.2381693810794
990
+ },
991
+ {
992
+ "pair_key": "construction|yolo26x|Qwen3.8",
993
+ "pair_label": "Construction / YOLO26x + Qwen3.8",
994
+ "task": "construction",
995
+ "method": "confidence",
996
+ "method_label": "置信度路由",
997
+ "nominal_budget": 0.99,
998
+ "seed_count": 1,
999
+ "test_metric": 0.2649306125590882,
1000
+ "test_metric_std": 0.0,
1001
+ "actual_test_call_fraction": 0.987017310252996,
1002
+ "actual_test_call_fraction_std": 0.0,
1003
+ "flops_gflops_per_input": 93666.33067394902,
1004
+ "latency_ms_per_input": 2597.9144591508966
1005
+ },
1006
+ {
1007
+ "pair_key": "construction|yolo26x|Qwen3.8",
1008
+ "pair_label": "Construction / YOLO26x + Qwen3.8",
1009
+ "task": "construction",
1010
+ "method": "confidence",
1011
+ "method_label": "置信度路由",
1012
+ "nominal_budget": 1.0,
1013
+ "seed_count": 1,
1014
+ "test_metric": 0.27001009067886245,
1015
+ "test_metric_std": 0.0,
1016
+ "actual_test_call_fraction": 1.0,
1017
+ "actual_test_call_fraction_std": 0.0,
1018
+ "flops_gflops_per_input": 94895.80944867841,
1019
+ "latency_ms_per_input": 2631.667776610426
1020
+ }
1021
+ ],
1022
+ "comparisons": {
1023
+ "grefcoco|instancevg|MiniCPM": {
1024
+ "both_vs_confidence_metric_delta_percentage_points": 6.175500288309744,
1025
+ "both_vs_confidence_call_delta_percentage_points": -40.80197374018093,
1026
+ "both_vs_confidence_flops_relative_percent": -78.21165529717908,
1027
+ "both_vs_confidence_latency_relative_percent": -76.66417123634449
1028
+ },
1029
+ "construction|yolo26x|Qwen3.8": {
1030
+ "both_vs_confidence_metric_delta_percentage_points": 4.1333041716827665,
1031
+ "both_vs_confidence_call_delta_percentage_points": -48.069241011984026,
1032
+ "both_vs_confidence_flops_relative_percent": -52.01415993620476,
1033
+ "both_vs_confidence_latency_relative_percent": -51.44759733081112
1034
+ }
1035
+ }
1036
+ }
releases/20260913_lr/paper_data/cost_20260911/fixed_budget_cost_performance.csv ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pair_key,pair_label,task,method,method_label,nominal_budget,seed_count,test_metric,test_metric_std,actual_test_call_fraction,actual_test_call_fraction_std,flops_gflops_per_input,latency_ms_per_input
2
+ grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,both,两项同时改,0.0,3,0.6695119530385957,0.0,0.0,0.0,7.503634432,16.318855201956467
3
+ grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,both,两项同时改,0.01,3,0.6743328701471798,0.0002504540415259137,0.021268751949635595,0.0024468276041848553,222.01478381055546,47.10392897831558
4
+ grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,both,两项同时改,0.05,3,0.6805811568310505,0.0004271568320976906,0.11976670542305912,0.0032185843726325893,1215.4397511547988,189.67302620038757
5
+ grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,both,两项同时改,0.1,3,0.6804866290446077,0.0005385595679416328,0.172031118547297,0.0033259813564282133,1742.5651488943693,265.32221416016074
6
+ grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,both,两项同时改,0.25,3,0.6794184650578038,0.00038210745789084077,0.24530905859777485,0.009454413833556307,2481.627562219953,371.38705501097667
7
+ grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,both,两项同时改,0.5,3,0.664568149807636,0.0016062493473305785,0.4310372534006371,0.0088737075287505,4354.834256882111,640.2160014811561
8
+ grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,both,两项同时改,0.75,3,0.6150355897115957,0.0013848240058531283,0.7057350008034863,0.006014701362190238,7125.364923966921,1037.8223321215987
9
+ grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,both,两项同时改,0.99,3,0.5069147075782926,9.357773432602366e-05,0.988051687793627,0.00048144255369636903,9972.73825167356,1446.4565603014837
10
+ grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,both,两项同时改,1.0,3,0.5000708958398321,0.0,1.0,0.0,10093.24584848,1463.7509306362917
11
+ grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,original,原方法,0.0,3,0.6695119530385957,0.0,0.0,0.0,7.503634432,16.318855201956467
12
+ grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,original,原方法,0.01,3,0.6742194368034484,0.0005415378002401663,0.04492905689627466,0.0027758676387843045,460.64652020812184,81.35061327263864
13
+ grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,original,原方法,0.05,3,0.679891103990018,0.0007177890142290943,0.10489748461560276,0.007840998690258661,1065.4726231670354,168.15083906695963
14
+ grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,original,原方法,0.1,3,0.680127423456125,0.0007907632127485806,0.16706840975904869,0.004046760924054268,1692.5125473727062,258.1390302790103
15
+ grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,original,原方法,0.25,3,0.6658726332605469,0.0048175982262131,0.3117620924670807,0.009206880232216604,3151.855731167172,467.5733077433342
16
+ grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,original,原方法,0.5,3,0.6273242019491629,0.009832598413097358,0.5310665570144345,0.009248673211785196,5363.704026981612,785.0016240151261
17
+ grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,original,原方法,0.75,3,0.5814404144098159,0.00858033252822142,0.7394719677849304,0.008502993761525655,7465.627276025616,1086.65430025841
18
+ grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,original,原方法,0.99,3,0.5100152189736173,0.0036056620177345906,0.978079006323909,0.009338895613008017,9872.15635718717,1432.0217812641245
19
+ grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,original,原方法,1.0,3,0.5000708958398321,0.0,1.0,0.0,10093.24584848,1463.7509306362917
20
+ grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,confidence,置信度路由,0.0,1,0.6695119530385957,0.0,0.0,0.0,7.503634432,16.318855201956467
21
+ grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,confidence,置信度路由,0.01,1,0.6686044862887446,0.0,0.002467175226157729,0.0,32.38692775991243,19.889923760014124
22
+ grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,confidence,置信度路由,0.05,1,0.6662791027422511,0.0,0.012534384482318578,0.0,133.9222051324287,34.46152534749078
23
+ grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,confidence,置信度路由,0.1,1,0.6645208859144145,0.0,0.0345971698380739,0.0,356.4417707544502,66.39590854483396
24
+ grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,confidence,置信度路由,0.25,1,0.6596999688058305,0.0,0.12477667810452883,0.0,1265.9690441195253,196.92462135659662
25
+ grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,confidence,置信度路由,0.5,1,0.6527521765022829,0.0,0.2345234381646485,0.0,2372.8465749128713,355.77560204260965
26
+ grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,confidence,置信度路由,0.75,1,0.567762243711539,0.0,0.7478093185491875,0.0,7549.715646582009,1098.722049178743
27
+ grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,confidence,置信度路由,0.99,1,0.5019425460114001,0.0,0.9908118991577575,0.0,10000.577031948465,1450.4517787649047
28
+ grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,confidence,置信度路由,1.0,1,0.5000708958398321,0.0,1.0,0.0,10093.24584848,1463.7509306362917
29
+ construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,both,两项同时改,0.0,3,0.18129624718318374,2.7755575615628914e-17,0.0,0.0,194.418697728,31.79686254823173
30
+ construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,both,两项同时改,0.01,3,0.1834834493716783,0.0012311365629078506,0.01187305814469596,0.0025254786233335643,1318.8138164976065,62.665281079596
31
+ construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,both,两项同时改,0.05,3,0.19244312207391248,0.0014149660525805208,0.04804704837993786,0.0025254786233335656,4744.541000786314,156.71298613777122
32
+ construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,both,两项同时改,0.1,3,0.2058996264309502,0.000756529123691676,0.10707944962272525,0.0037002512207130658,10334.991497846413,310.1896091161431
33
+ construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,both,两项同时改,0.25,3,0.25617136079673564,0.0020077364088740926,0.26498002663115844,0.008293336414138841,25288.39574092258,720.7107265940061
34
+ construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,both,两项同时改,0.5,3,0.29980106360416753,0.002221016752728669,0.49511762094984463,0.00440231567351916,47082.74598698018,1319.038764295403
35
+ construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,both,两项同时改,0.75,3,0.28788636673929996,0.0011284209510194398,0.7496671105193076,0.005429263125932131,71188.93666415288,1980.8345784164278
36
+ construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,both,两项同时改,0.99,3,0.27239176103160984,0.0005578690368974412,0.9862405681313805,0.0023116585281845767,93592.7721147772,2595.8950299011817
37
+ construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,both,两项同时改,1.0,3,0.27001009067886245,0.0,1.0,0.0,94895.80944867841,2631.667776610426
38
+ construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,original,原方法,0.0,3,0.18129624718318374,2.7755575615628914e-17,0.0,0.0,194.418697728,31.79686254823173
39
+ construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,original,原方法,0.01,3,0.18300903853620068,0.000992883110506064,0.010319573901464712,0.0032954377285658006,1171.6966981539194,58.626422580165155
40
+ construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,original,原方法,0.05,3,0.19493527207894204,0.0032238119689520237,0.049822458943630715,0.011397380483895993,4912.674850321956,161.32882442283505
41
+ construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,original,原方法,0.1,3,0.20346296196880112,0.004110692938093332,0.09576120727918332,0.006259345218007303,9263.138207056696,280.76364004886125
42
+ construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,original,原方法,0.25,3,0.24461408052105227,0.0022654085644492608,0.2533288948069241,0.0017823322685540256,24185.01735334493,690.419287848275
43
+ construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,original,原方法,0.5,3,0.29849806777867033,0.0015269090065487179,0.4957833999112295,0.00518804692745715,47145.796180556055,1320.7697036523023
44
+ construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,original,原方法,0.75,3,0.28532383356176916,0.0029913445567370696,0.7654238792720817,0.006553405708048556,72681.1245787817,2021.8001431963692
45
+ construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,original,原方法,0.99,3,0.27053969734098926,0.0014164533171887024,0.9897913892587661,0.0027895705925380728,93929.03981384847,2605.126706471309
46
+ construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,original,原方法,1.0,3,0.27001009067886245,0.0,1.0,0.0,94895.80944867841,2631.667776610426
47
+ construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,confidence,置信度路由,0.0,1,0.18129624718318377,0.0,0.0,0.0,194.418697728,31.79686254823173
48
+ construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,confidence,置信度路由,0.01,1,0.18129624718318377,0.0,0.0,0.0,194.418697728,31.79686254823173
49
+ construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,confidence,置信度路由,0.05,1,0.18129624718318377,0.0,0.0,0.0,194.418697728,31.79686254823173
50
+ construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,confidence,置信度路由,0.1,1,0.18129624718318377,0.0,0.0,0.0,194.418697728,31.79686254823173
51
+ construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,confidence,置信度路由,0.25,1,0.18026802885144627,0.0,0.24800266311584554,0.0,23680.615804738005,676.5717729930834
52
+ construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,confidence,置信度路由,0.5,1,0.18015785955296285,0.0,0.5013315579227696,0.0,47671.21446035493,1335.1941982931266
53
+ construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,confidence,置信度路由,0.75,1,0.22103445955201576,0.0,0.7486684420772304,0.0,71094.36137378908,1978.2381693810794
54
+ construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,confidence,置信度路由,0.99,1,0.2649306125590882,0.0,0.987017310252996,0.0,93666.33067394902,2597.9144591508966
55
+ construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,confidence,置信度路由,1.0,1,0.27001009067886245,0.0,1.0,0.0,94895.80944867841,2631.667776610426
releases/20260913_lr/paper_data/cost_20260911/full_cost_performance_curves.csv ADDED
The diff for this file is too large to render. See raw diff
 
releases/20260913_lr/paper_data/cost_20260911/grefcoco_router.json ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "component": "router",
3
+ "expert": "instancevg",
4
+ "task": "grefcoco",
5
+ "endpoint": "MiniCPM",
6
+ "condition": "both",
7
+ "seed": 42,
8
+ "device": "cuda:0",
9
+ "checkpoint": "/mnt/HithinkOmniSSD/user_workspace/caisihang/research/ICASSP_ModelCollaboration/outputs/router_method_improvement_20260911_r1/conditions/both_instancevg_MiniCPM_seed42/best.pt",
10
+ "sample_count": 1024,
11
+ "batch_size": 256,
12
+ "row_count_loaded": 35263,
13
+ "mean_latency_ms_per_sample": 0.001830718741985038,
14
+ "median_latency_ms_per_sample": 0.0018171250121667981,
15
+ "latency_ms_per_sample": [
16
+ 0.001949499943293631,
17
+ 0.0018388750031590462,
18
+ 0.00179537502117455,
19
+ 0.0017391250003129244
20
+ ],
21
+ "flops_counted_per_batch": 0,
22
+ "gflops_counted_per_sample": 0.0,
23
+ "operators": {},
24
+ "flops_scope": "GPU FlopCounterMode for the router forward; multiply-add counted as 2 FLOPs.",
25
+ "latency_scope": "A800 CUDA forward latency on cached router features, excluding feature loading and model load.",
26
+ "completed": true
27
+ }
releases/20260913_lr/paper_data/cost_20260911/instancevg_specialist.json ADDED
@@ -0,0 +1,134 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "component": "specialist",
3
+ "expert": "instancevg",
4
+ "task": "grefcoco",
5
+ "device": "cuda:0",
6
+ "checkpoint": "/mnt/HithinkOmniSSD/user_workspace/caisihang/research/ICASSP_ModelCollaboration/models/grefcoco/instancevg/InstanceVG-grefcoco.pth",
7
+ "sample_count": 64,
8
+ "batch_size": 16,
9
+ "sample_ids": [
10
+ "testA-30844-84861",
11
+ "testB-44353-122077",
12
+ "testB-36889-101643",
13
+ "testA-49055-135087",
14
+ "testA-81245-223633",
15
+ "testB-37669-103833",
16
+ "testA-5038-13943",
17
+ "testB-79646-219274",
18
+ "testA-469-1309",
19
+ "testB-46644-128438",
20
+ "testA-24562-67515",
21
+ "testA-47070-129599",
22
+ "testA-54554-150141",
23
+ "testB-12382-34099",
24
+ "testB-23014-63279",
25
+ "testB-79647-219275",
26
+ "testB-69479-191564",
27
+ "testA-15582-42797",
28
+ "testB-56692-156000",
29
+ "testB-42726-117660",
30
+ "testA-80455-221494",
31
+ "testB-33115-91174",
32
+ "testA-62940-173455",
33
+ "testB-31494-86670",
34
+ "testA-54586-150233",
35
+ "testA-81445-224177",
36
+ "testB-8417-23141",
37
+ "testA-24438-67190",
38
+ "testA-59285-163255",
39
+ "testA-73258-201893",
40
+ "testB-945-2633",
41
+ "testA-54307-149426",
42
+ "testB-60557-166822",
43
+ "testA-81622-224665",
44
+ "testA-69466-191525",
45
+ "testA-60731-167321",
46
+ "testA-7857-21644",
47
+ "testA-44142-121499",
48
+ "testB-66162-182407",
49
+ "testA-31764-87406",
50
+ "testB-38637-106482",
51
+ "testB-52921-145709",
52
+ "testB-65796-181394",
53
+ "testB-2936-8113",
54
+ "testA-79438-218690",
55
+ "testA-57397-158016",
56
+ "testB-23920-65743",
57
+ "testB-15419-42358",
58
+ "testA-10078-27739",
59
+ "testA-43890-120793",
60
+ "testA-55116-151689",
61
+ "testB-27487-75582",
62
+ "testA-26856-73830",
63
+ "testB-50845-140107",
64
+ "testA-29199-80374",
65
+ "testB-65935-181778",
66
+ "testA-17736-48684",
67
+ "testB-69477-191553",
68
+ "testB-22290-61227",
69
+ "testA-58476-160950",
70
+ "testB-36265-99859",
71
+ "testA-5734-15837",
72
+ "testB-67952-187353",
73
+ "testB-47054-129555"
74
+ ],
75
+ "model_load_seconds_excluded_from_latency": true,
76
+ "forward_latency_ms_per_sample": [
77
+ 11.21629810333252,
78
+ 11.368107795715332,
79
+ 11.16999626159668,
80
+ 11.233466148376465
81
+ ],
82
+ "end_to_end_latency_ms_per_sample": [
83
+ 16.296884561597835,
84
+ 16.604527561867144,
85
+ 15.993722627172247,
86
+ 16.3729631822207
87
+ ],
88
+ "mean_forward_latency_ms_per_sample": 11.246967077255249,
89
+ "median_forward_latency_ms_per_sample": 11.224882125854492,
90
+ "mean_end_to_end_latency_ms_per_sample": 16.31702448321448,
91
+ "median_end_to_end_latency_ms_per_sample": 16.334923871909268,
92
+ "flop_profiles": [
93
+ {
94
+ "batch_size": 16,
95
+ "flops_counted": 120042881024,
96
+ "gflops_counted_per_sample": 7.502680064,
97
+ "operators": {
98
+ "aten.bmm": 120010113024,
99
+ "aten.baddbmm": 32768000
100
+ }
101
+ },
102
+ {
103
+ "batch_size": 16,
104
+ "flops_counted": 120042881024,
105
+ "gflops_counted_per_sample": 7.502680064,
106
+ "operators": {
107
+ "aten.bmm": 120010113024,
108
+ "aten.baddbmm": 32768000
109
+ }
110
+ },
111
+ {
112
+ "batch_size": 16,
113
+ "flops_counted": 120042881024,
114
+ "gflops_counted_per_sample": 7.502680064,
115
+ "operators": {
116
+ "aten.bmm": 120010113024,
117
+ "aten.baddbmm": 32768000
118
+ }
119
+ },
120
+ {
121
+ "batch_size": 16,
122
+ "flops_counted": 120042881024,
123
+ "gflops_counted_per_sample": 7.502680064,
124
+ "operators": {
125
+ "aten.bmm": 120010113024,
126
+ "aten.baddbmm": 32768000
127
+ }
128
+ }
129
+ ],
130
+ "mean_counted_gflops_per_sample": 7.502680064,
131
+ "flops_scope": "GPU FlopCounterMode dense products, convolutions and supported attention; multiply-add counted as 2 FLOPs. Elementwise, normalization, custom ops and postprocessing may be missing.",
132
+ "latency_scope": "A800 wall-clock CUDA forward latency and end-to-end latency including image read/preprocess, excluding model load; batch processing uses the recorded batch size.",
133
+ "completed": true
134
+ }
releases/20260913_lr/paper_data/cost_20260911/make_cost_analysis_20260911.py ADDED
@@ -0,0 +1,457 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Build the A800 cost--performance supplement for two representative pairs.
2
+
3
+ The script reads the frozen test curves and the measured A800 component records.
4
+ It does not select a test-set operating point: the selected points are copied from
5
+ the validation-locked records, and all curve values retain their nominal budget.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import argparse
11
+ import csv
12
+ import json
13
+ from pathlib import Path
14
+
15
+ import matplotlib.pyplot as plt
16
+ import numpy as np
17
+
18
+
19
+ PAIR_FILES = {
20
+ "grefcoco|instancevg|MiniCPM": {
21
+ "label": "gRefCOCO / InstanceVG + MiniCPM",
22
+ "specialist": "instancevg_specialist.json",
23
+ "router": "grefcoco_router.json",
24
+ "large_latency": "minicpm_grefcoco.json",
25
+ "large_flops": "minicpm_grefcoco_flops.json",
26
+ },
27
+ "construction|yolo26x|Qwen3.8": {
28
+ "label": "Construction / YOLO26x + Qwen3.8",
29
+ "specialist": "yolo26x_specialist.json",
30
+ "router": "construction_router.json",
31
+ "large_latency": "qwen_construction.json",
32
+ "large_flops": "qwen_construction_flops.json",
33
+ },
34
+ }
35
+
36
+ METHOD_LABELS = {
37
+ "both": "两项同时改",
38
+ "original": "原方法",
39
+ "confidence": "置信度路由",
40
+ }
41
+ PLOT_LABELS = {"both": "Two changes", "original": "Original", "confidence": "Confidence"}
42
+ PLOT_METHODS = ("both", "original", "confidence")
43
+ PLOT_COLORS = {"both": "#2166ac", "original": "#d6604d", "confidence": "#252525"}
44
+ PLOT_STYLES = {"both": "-", "original": "--", "confidence": ":"}
45
+
46
+
47
+ def read_json(path: Path):
48
+ return json.loads(path.read_text(encoding="utf-8"))
49
+
50
+
51
+ def write_json(path: Path, value) -> None:
52
+ path.write_text(json.dumps(value, ensure_ascii=False, indent=2), encoding="utf-8")
53
+
54
+
55
+ def mean_std(values):
56
+ values = np.asarray(values, dtype=float)
57
+ return float(values.mean()), float(values.std(ddof=0))
58
+
59
+
60
+ def router_flops(architecture: dict) -> tuple[int, dict[str, int]]:
61
+ d = int(architecture["representation_dim"])
62
+ branch_dims = architecture["input_dimensions"]
63
+ branch_costs = {}
64
+ for branch in architecture["branches"]:
65
+ input_key = "image_flat_plus_valid" if branch == "image" else branch
66
+ branch_costs[branch] = 2 * int(branch_dims[input_key]) * d
67
+ head_input = len(architecture["branches"]) * d
68
+ branch_costs["head_input_to_hidden"] = 2 * head_input * int(architecture["head_hidden_dim"])
69
+ branch_costs["head_hidden_to_classes"] = 2 * int(architecture["head_hidden_dim"]) * int(architecture["class_count"])
70
+ return sum(branch_costs.values()), branch_costs
71
+
72
+
73
+ def build_cost_components(root: Path, architectures: list[dict]) -> dict[str, dict]:
74
+ components = {}
75
+ for architecture in architectures:
76
+ key = architecture["pair_key"]
77
+ files = PAIR_FILES[key]
78
+ specialist = read_json(root / files["specialist"])
79
+ router = read_json(root / files["router"])
80
+ large_latency = read_json(root / files["large_latency"])
81
+ large_flops = read_json(root / files["large_flops"])
82
+ router_flop_count, router_breakdown = router_flops(architecture)
83
+ completion_tokens = [
84
+ output["completion_tokens"]
85
+ for batch in large_latency["batches"]
86
+ for output in batch["outputs"]
87
+ ]
88
+ prefill = float(large_flops["mean_prefill_gflops_counted"])
89
+ decode = float(large_flops["mean_decode_one_token_gflops_counted"])
90
+ mean_tokens = float(large_latency["mean_completion_tokens"])
91
+ large_estimated = prefill + decode * mean_tokens
92
+ components[key] = {
93
+ "pair_key": key,
94
+ "task": architecture["task"],
95
+ "expert": architecture["expert"],
96
+ "endpoint": architecture["endpoint"],
97
+ "label": PAIR_FILES[key]["label"],
98
+ "specialist_gflops_per_input": float(specialist["mean_counted_gflops_per_sample"]),
99
+ "specialist_forward_latency_ms": float(specialist["mean_forward_latency_ms_per_sample"]),
100
+ "specialist_end_to_end_latency_ms": float(specialist["mean_end_to_end_latency_ms_per_sample"]),
101
+ "specialist_sample_count": int(specialist["sample_count"]),
102
+ "router_gflops_per_input": router_flop_count / 1e9,
103
+ "router_flop_count": router_flop_count,
104
+ "router_flop_breakdown": router_breakdown,
105
+ "router_measured_gflops_per_input": float(router["gflops_counted_per_sample"]),
106
+ "router_latency_ms": float(router["mean_latency_ms_per_sample"]),
107
+ "router_sample_count": int(router["sample_count"]),
108
+ "large_prefill_gflops": prefill,
109
+ "large_decode_one_token_gflops": decode,
110
+ "large_mean_completion_tokens": mean_tokens,
111
+ "large_completion_token_min": int(min(completion_tokens)),
112
+ "large_completion_token_max": int(max(completion_tokens)),
113
+ "large_estimated_full_generation_gflops": large_estimated,
114
+ "large_batch_wall_latency_ms_per_input": float(large_latency["mean_batch_wall_ms_per_sample"]),
115
+ "large_measured_sample_count": int(large_latency["measured_count"]),
116
+ "large_flops_probe_count": int(large_flops["sample_count"]),
117
+ "flops_formula": "specialist + router + actual_call_fraction * (prefill + mean_completion_tokens * one_token_decode)",
118
+ "latency_formula": "specialist end-to-end + router + actual_call_fraction * large-model batch-wall latency",
119
+ "specialist_flops_scope": specialist["flops_scope"],
120
+ "specialist_latency_scope": specialist["latency_scope"],
121
+ "large_flops_scope": large_flops["scope"],
122
+ "large_latency_scope": large_latency["latency_scope"],
123
+ }
124
+ return components
125
+
126
+
127
+ def load_relevant_groups(results: dict, pair_key: str) -> dict[str, list[dict]]:
128
+ task, expert, endpoint = pair_key.split("|")
129
+ groups = {
130
+ method: []
131
+ for method in PLOT_METHODS
132
+ }
133
+ for group in results["groups"]:
134
+ if (group["task"], group["expert"], group["endpoint"]) != (task, expert, endpoint):
135
+ continue
136
+ if group["method"] in groups:
137
+ groups[group["method"]].append(group)
138
+ for method, entries in groups.items():
139
+ if not entries:
140
+ raise ValueError(f"missing {method} group for {pair_key}")
141
+ return groups
142
+
143
+
144
+ def curve_rows(groups: dict[str, list[dict]], pair_key: str, components: dict) -> list[dict]:
145
+ rows = []
146
+ for method, entries in groups.items():
147
+ per_budget = {}
148
+ for group in entries:
149
+ curve = group["deployments"]["endpoint"]["test_curve"]
150
+ for point in curve:
151
+ budget = round(float(point["budget"]), 10)
152
+ per_budget.setdefault(budget, []).append(point)
153
+ for budget in sorted(per_budget):
154
+ points = per_budget[budget]
155
+ metrics = [float(point["metric"]) for point in points]
156
+ calls = [float(point["actual_fraction"]) for point in points]
157
+ metric_mean, metric_std = mean_std(metrics)
158
+ call_mean, call_std = mean_std(calls)
159
+ component = components[pair_key]
160
+ rows.append({
161
+ "pair_key": pair_key,
162
+ "pair_label": component["label"],
163
+ "task": component["task"],
164
+ "method": method,
165
+ "method_label": METHOD_LABELS[method],
166
+ "nominal_budget": budget,
167
+ "seed_count": len(points),
168
+ "test_metric": metric_mean,
169
+ "test_metric_std": metric_std,
170
+ "actual_test_call_fraction": call_mean,
171
+ "actual_test_call_fraction_std": call_std,
172
+ "flops_gflops_per_input": component["specialist_gflops_per_input"] + component["router_gflops_per_input"] + call_mean * component["large_estimated_full_generation_gflops"],
173
+ "latency_ms_per_input": component["specialist_end_to_end_latency_ms"] + component["router_latency_ms"] + call_mean * component["large_batch_wall_latency_ms_per_input"],
174
+ })
175
+ return rows
176
+
177
+
178
+ def selected_rows(groups: dict[str, list[dict]], pair_key: str, components: dict) -> list[dict]:
179
+ rows = []
180
+ component = components[pair_key]
181
+ for method, entries in groups.items():
182
+ points = [entry["deployments"]["endpoint"]["test_point"] for entry in entries]
183
+ metrics = [float(point["metric"]) for point in points]
184
+ calls = [float(point["actual_fraction"]) for point in points]
185
+ budgets = [float(point["budget"]) for point in points]
186
+ metric_mean, metric_std = mean_std(metrics)
187
+ call_mean, call_std = mean_std(calls)
188
+ rows.append({
189
+ "pair_key": pair_key,
190
+ "pair_label": component["label"],
191
+ "task": component["task"],
192
+ "method": method,
193
+ "method_label": METHOD_LABELS[method],
194
+ "seed_count": len(points),
195
+ "selected_nominal_budget_mean": float(np.mean(budgets)),
196
+ "selected_nominal_budgets": budgets,
197
+ "test_metric": metric_mean,
198
+ "test_metric_std": metric_std,
199
+ "actual_test_call_fraction": call_mean,
200
+ "actual_test_call_fraction_std": call_std,
201
+ "flops_gflops_per_input": component["specialist_gflops_per_input"] + component["router_gflops_per_input"] + call_mean * component["large_estimated_full_generation_gflops"],
202
+ "latency_ms_per_input": component["specialist_end_to_end_latency_ms"] + component["router_latency_ms"] + call_mean * component["large_batch_wall_latency_ms_per_input"],
203
+ })
204
+ return rows
205
+
206
+
207
+ def fixed_budget_rows(curves: list[dict], budgets=(0.0, 0.01, 0.05, 0.10, 0.25, 0.50, 0.75, 0.99, 1.0)) -> list[dict]:
208
+ wanted = {round(float(budget), 10) for budget in budgets}
209
+ return [row for row in curves if round(float(row["nominal_budget"]), 10) in wanted]
210
+
211
+
212
+ def write_csv(path: Path, rows: list[dict]) -> None:
213
+ if not rows:
214
+ raise ValueError(f"no rows for {path}")
215
+ fields = list(rows[0])
216
+ with path.open("w", encoding="utf-8", newline="") as handle:
217
+ writer = csv.DictWriter(handle, fieldnames=fields)
218
+ writer.writeheader()
219
+ writer.writerows(rows)
220
+
221
+
222
+ def make_plot(curves: list[dict], selected: list[dict], out_path: Path, cost_field: str, x_label: str, log_x: bool) -> None:
223
+ fig, axes = plt.subplots(1, 2, figsize=(7.2, 2.9), constrained_layout=True)
224
+ pair_keys = list(PAIR_FILES)
225
+ for index, (ax, pair_key) in enumerate(zip(axes, pair_keys)):
226
+ pair_rows = [row for row in curves if row["pair_key"] == pair_key]
227
+ all_y = []
228
+ for method in PLOT_METHODS:
229
+ method_rows = [row for row in pair_rows if row["method"] == method]
230
+ x = np.asarray([row[cost_field] for row in method_rows], dtype=float)
231
+ y = 100 * np.asarray([row["test_metric"] for row in method_rows], dtype=float)
232
+ y_std = 100 * np.asarray([row["test_metric_std"] for row in method_rows], dtype=float)
233
+ all_y.extend((y - y_std).tolist())
234
+ all_y.extend((y + y_std).tolist())
235
+ label = PLOT_LABELS[method]
236
+ ax.plot(x, y, PLOT_STYLES[method], color=PLOT_COLORS[method], linewidth=1.25, label=label)
237
+ if method in ("both", "original") and np.any(y_std > 0):
238
+ ax.fill_between(x, y - y_std, y + y_std, color=PLOT_COLORS[method], alpha=0.10, linewidth=0)
239
+ selected_row = next(row for row in selected if row["pair_key"] == pair_key and row["method"] == method)
240
+ ax.scatter(
241
+ [selected_row[cost_field]],
242
+ [100 * selected_row["test_metric"]],
243
+ color=PLOT_COLORS[method],
244
+ marker="*",
245
+ s=42,
246
+ edgecolor="white",
247
+ linewidth=0.45,
248
+ zorder=5,
249
+ )
250
+ y_min, y_max = min(all_y), max(all_y)
251
+ y_pad = max(0.05 * (y_max - y_min), 0.2)
252
+ ax.set_ylim(y_min - y_pad, y_max + 2.0 * y_pad)
253
+ task_label = "gRefCOCO" if PAIR_FILES[pair_key]["label"].startswith("gRefCOCO") else "Construction"
254
+ ax.text(0.02, 0.98, f"({'ab'[index]}) {task_label}", transform=ax.transAxes, ha="left", va="top", fontsize=8.5)
255
+ ax.set_xlabel(x_label)
256
+ ax.set_ylabel("Test performance (%)")
257
+ ax.grid(True, color="#d9d9d9", linewidth=0.45, alpha=0.65)
258
+ ax.spines["top"].set_visible(False)
259
+ ax.spines["right"].set_visible(False)
260
+ if log_x:
261
+ ax.set_xscale("log")
262
+ ax.tick_params(labelsize=8)
263
+ handles, labels = axes[0].get_legend_handles_labels()
264
+ fig.legend(handles, labels, loc="lower center", bbox_to_anchor=(0.5, -0.045), ncol=3, frameon=False, fontsize=8)
265
+ fig.savefig(out_path, format="pdf", bbox_inches="tight", pad_inches=0.04)
266
+ fig.savefig(out_path.with_suffix(".png"), format="png", dpi=300, bbox_inches="tight", pad_inches=0.04)
267
+ plt.close(fig)
268
+
269
+
270
+ def fmt(value: float, digits=3) -> str:
271
+ return f"{value:.{digits}f}"
272
+
273
+
274
+ def pct(value: float, digits=2) -> str:
275
+ return f"{100 * value:.{digits}f}%"
276
+
277
+
278
+ def report_markdown(components: dict, selected: list[dict], fixed: list[dict], root: Path, summary: dict) -> str:
279
+ selected_by_key = {(row["pair_key"], row["method"]): row for row in selected}
280
+ lines = [
281
+ "# A800成本—性能补充报告(2026-09-11)",
282
+ "",
283
+ "## 1. 做了什么",
284
+ "",
285
+ "- 在A800上补测了正文中两个代表性模型对:gRefCOCO的InstanceVG + MiniCPM,以及Construction的YOLO26x + Qwen3.8。",
286
+ "- 分别实测了小模型端、路由器端和大模型端的延迟与可计数运算量;大模型端延迟采用批处理均摊后的真实墙钟时间。",
287
+ "- 从已经锁定的测试曲线中提取代表性调用预算,按测试集实际调用率换算每个输入的成本,并生成完整曲线。没有用测试集重新挑选模型、阈值或预算。",
288
+ "",
289
+ "## 2. 为什么做",
290
+ "",
291
+ "正文原有成本图只是代理估算,缺少InstanceVG、完整解码和真实延迟。此次补测用于回答一个更直接的问题:两项同时改相对置信度路由的性能提升,是否值得它增加的计算和时间成本。",
292
+ "",
293
+ "## 3. 原始实测成本",
294
+ "",
295
+ "| 模型对 | 小模型端到端延迟(毫秒/样本) | 小模型可计数运算量(GFLOPs/样本) | 路由器延迟(毫秒/样本) | 路由器稠密运算量(GFLOPs/样本) | 大模型调用延迟(毫秒/样本) | 大模型完整生成估算(GFLOPs/调用) |",
296
+ "|---|---:|---:|---:|---:|---:|---:|",
297
+ ]
298
+ for key, component in components.items():
299
+ lines.append(
300
+ f"| {component['label']} | {fmt(component['specialist_end_to_end_latency_ms'])} | {fmt(component['specialist_gflops_per_input'])} | {fmt(component['router_latency_ms'], 6)} | {fmt(component['router_gflops_per_input'], 6)} | {fmt(component['large_batch_wall_latency_ms_per_input'])} | {fmt(component['large_estimated_full_generation_gflops'])} |"
301
+ )
302
+ lines.extend([
303
+ "",
304
+ "说明:GFLOPs按乘加计2次运算,只覆盖计数器能识别的稠密乘法、卷积和注意力等算子;路由器的计数器返回0,因此按已加载检查点的线性层结构精确计算了稠密乘加量。大模型完整生成GFLOPs为“预填充GFLOPs + 平均生成长度 × 单Token解码GFLOPs”的估算,不把未计数算子冒充成完整硬件计数。",
305
+ "",
306
+ "## 4. 验证集锁定点在测试集上的结果",
307
+ "",
308
+ "下表的均值和标准差只在三个随机种子之间统计。成本使用对应的平均实际调用率;星号表示验证集选出的运行点。",
309
+ "",
310
+ "| 模型对 | 方法 | 测试性能 | 实际调用率 | 估算GFLOPs/输入 | 端到端延迟(毫秒/输入) |",
311
+ "|---|---|---:|---:|---:|---:|",
312
+ ])
313
+ for key in components:
314
+ for method in ("both", "original", "confidence"):
315
+ row = selected_by_key[(key, method)]
316
+ metric = f"{100 * row['test_metric']:.2f}±{100 * row['test_metric_std']:.2f}%"
317
+ calls = f"{100 * row['actual_test_call_fraction']:.2f}±{100 * row['actual_test_call_fraction_std']:.2f}%"
318
+ lines.append(f"| {row['pair_label']} | {row['method_label']}* | {metric} | {calls} | {fmt(row['flops_gflops_per_input'])} | {fmt(row['latency_ms_per_input'])} |")
319
+ lines.extend([
320
+ "",
321
+ "### 两项同时改相对置信度路由",
322
+ "",
323
+ "| 模型对 | 性能变化 | 调用率变化 | 估算GFLOPs变化 | 实际延迟变化 |",
324
+ "|---|---:|---:|---:|---:|",
325
+ ])
326
+ for key in components:
327
+ learned = selected_by_key[(key, "both")]
328
+ confidence = selected_by_key[(key, "confidence")]
329
+ metric_delta = 100 * (learned["test_metric"] - confidence["test_metric"])
330
+ call_delta = 100 * (learned["actual_test_call_fraction"] - confidence["actual_test_call_fraction"])
331
+ flop_delta = 100 * (learned["flops_gflops_per_input"] / confidence["flops_gflops_per_input"] - 1)
332
+ latency_delta = 100 * (learned["latency_ms_per_input"] / confidence["latency_ms_per_input"] - 1)
333
+ lines.append(f"| {learned['pair_label']} | {metric_delta:+.2f} 个百分点 | {call_delta:+.2f} 个百分点 | {flop_delta:+.1f}% | {latency_delta:+.1f}% |")
334
+ summary[key]["both_vs_confidence"] = {
335
+ "metric_delta_percentage_points": metric_delta,
336
+ "call_delta_percentage_points": call_delta,
337
+ "flops_relative_percent": flop_delta,
338
+ "latency_relative_percent": latency_delta,
339
+ }
340
+ lines.extend([
341
+ "",
342
+ "直白地说:两项同时改在这两个代表性模型对上都提高了测试性能,同时减少了实际调用,因此计算量和延迟也一起下降。原因不是路由器本身更快,而是它让大模型少被调用;路由器自身成本相对很小。",
343
+ "",
344
+ "## 5. 固定名义调用预算的代表点",
345
+ "",
346
+ "名义预算是验证阶段设定的阈值预算;阈值原样转到测试集后,实际调用率会有偏差。完整的99个阈值点和端点均在CSV中。",
347
+ "",
348
+ "| 模型对 | 名义预算 | 两项同时改:性能 / 实际调用 | 置信度:性能 / 实际调用 | 两项同时改:GFLOPs/输入 | 两项同时改:毫秒/输入 |",
349
+ "|---|---:|---:|---:|---:|---:|",
350
+ ])
351
+ for key in components:
352
+ rows = {(row["nominal_budget"], row["method"]): row for row in fixed if row["pair_key"] == key}
353
+ for budget in (0.25, 0.50):
354
+ learned = rows[(budget, "both")]
355
+ confidence = rows[(budget, "confidence")]
356
+ lines.append(
357
+ f"| {components[key]['label']} | {100 * budget:.0f}% | {100 * learned['test_metric']:.2f}% / {100 * learned['actual_test_call_fraction']:.2f}% | {100 * confidence['test_metric']:.2f}% / {100 * confidence['actual_test_call_fraction']:.2f}% | {fmt(learned['flops_gflops_per_input'])} | {fmt(learned['latency_ms_per_input'])} |"
358
+ )
359
+ lines.extend([
360
+ "",
361
+ "## 6. 结果怎么理解",
362
+ "",
363
+ "1. gRefCOCO的InstanceVG + MiniCPM是这两组中相对置信度提升更明显的一组;Construction的YOLO26x + Qwen3.8也有稳定的正向变化。",
364
+ "2. 这里的成本是按真实测试调用率换算的平均每输入成本,不是把名义预算当成实际调用率。",
365
+ "3. 大模型延迟是A800、批量4下的均摊值,模型加载和预热没有计入;实际系统的并发度、批量大小和请求排队会改变墙钟延迟。",
366
+ "4. InstanceVG使用了远端已有的MMCV等价算子兼容路径,因为原生detrex扩展不可用;这不改变本次模型输出链路,但应在复现实验时保留该环境说明。",
367
+ "5. 本补充不修改正文,也不覆盖旧实验结果;它是针对两个代表性模型对的成本实测和成本—性能换算。",
368
+ "",
369
+ "## 7. 文件在哪里",
370
+ "",
371
+ f"- 原始A800测量记录:`{root.as_posix()}`目录下的8个JSON文件。",
372
+ f"- 完整成本—性能曲线数据:`{(root / 'full_cost_performance_curves.csv').as_posix()}`。",
373
+ f"- 固定预算代表点:`{(root / 'fixed_budget_cost_performance.csv').as_posix()}`。",
374
+ f"- FLOPs—性能图:`{(root / 'flops_performance.pdf').as_posix()}`;实际延迟—性能图:`{(root / 'latency_performance.pdf').as_posix()}`。",
375
+ f"- 可复现脚本:`{(root / 'make_cost_analysis_20260911.py').as_posix()}`;插入论文的LaTeX片段:`{(root / 'latex_includes.tex').as_posix()}`。",
376
+ ])
377
+ return "\n".join(lines) + "\n"
378
+
379
+
380
+ def latex_includes() -> str:
381
+ return r"""% A800 cost--performance supplement (generated 2026-09-11)
382
+ \begin{figure*}[t]
383
+ \centering
384
+ \includegraphics[width=0.98\textwidth]{dse_results/method_improvement_20260911/cost_analysis_20260911/flops_performance.pdf}
385
+ \caption{A800 counted-FLOPs--performance curves for the two representative model pairs. Each point uses a validation-calibrated nominal budget transferred unchanged to test; the horizontal coordinate uses the resulting actual test call rate. The star marks the validation-selected operating point. Large-model full-generation FLOPs are estimated from measured prefill FLOPs, one-token decode FLOPs and the measured mean completion length; unsupported operations are not included.}
386
+ \label{fig:a800-flops-performance}
387
+ \end{figure*}
388
+
389
+ \begin{figure*}[t]
390
+ \centering
391
+ \includegraphics[width=0.98\textwidth]{dse_results/method_improvement_20260911/cost_analysis_20260911/latency_performance.pdf}
392
+ \caption{A800 end-to-end latency--performance curves. Specialist and router latency are always paid; the large-model term is weighted by the actual test call rate and uses the measured batch-wall latency per input. Model loading and warm-up are excluded.}
393
+ \label{fig:a800-latency-performance}
394
+ \end{figure*}
395
+ """
396
+
397
+
398
+ def main() -> None:
399
+ parser = argparse.ArgumentParser()
400
+ parser.add_argument("--experiment-dir", type=Path, default=Path(__file__).resolve().parent.parent)
401
+ parser.add_argument("--output-dir", type=Path, default=Path(__file__).resolve().parent)
402
+ args = parser.parse_args()
403
+ root = args.output_dir.resolve()
404
+ experiment_dir = args.experiment_dir.resolve()
405
+ root.mkdir(parents=True, exist_ok=True)
406
+
407
+ architectures = read_json(root / "router_architecture.json")
408
+ architecture_keys = {item["pair_key"] for item in architectures}
409
+ if architecture_keys != set(PAIR_FILES):
410
+ raise ValueError("router architecture keys do not match selected pairs")
411
+ results = read_json(experiment_dir / "test_results.json")
412
+ components = build_cost_components(root, architectures)
413
+ all_curves = []
414
+ all_selected = []
415
+ for pair_key in PAIR_FILES:
416
+ groups = load_relevant_groups(results, pair_key)
417
+ all_curves.extend(curve_rows(groups, pair_key, components))
418
+ all_selected.extend(selected_rows(groups, pair_key, components))
419
+ all_fixed = fixed_budget_rows(all_curves)
420
+
421
+ write_csv(root / "full_cost_performance_curves.csv", all_curves)
422
+ write_csv(root / "fixed_budget_cost_performance.csv", all_fixed)
423
+ make_plot(all_curves, all_selected, root / "flops_performance.pdf", "flops_gflops_per_input", "Estimated GFLOPs / input", True)
424
+ make_plot(all_curves, all_selected, root / "latency_performance.pdf", "latency_ms_per_input", "End-to-end latency (ms / input)", False)
425
+ (root / "latex_includes.tex").write_text(latex_includes(), encoding="utf-8")
426
+
427
+ summary = {
428
+ "status": "complete",
429
+ "protocol": {
430
+ "test_results": str((experiment_dir / "test_results.json").resolve()),
431
+ "curves_use": "validation-calibrated thresholds transferred unchanged to test",
432
+ "cost_uses": "actual test call fraction",
433
+ "flop_counting": "multiply-add counted as 2 FLOPs; unsupported operations excluded",
434
+ "large_generation_flops": "mean prefill + mean completion tokens * one-token decode probe",
435
+ "seed_aggregation": "mean and population standard deviation across three seeds for learned/original methods",
436
+ },
437
+ "components": components,
438
+ "selected_points": all_selected,
439
+ "fixed_budget_points": all_fixed,
440
+ }
441
+ for key in components:
442
+ summary.setdefault("comparisons", {})
443
+ learned = next(row for row in all_selected if row["pair_key"] == key and row["method"] == "both")
444
+ confidence = next(row for row in all_selected if row["pair_key"] == key and row["method"] == "confidence")
445
+ summary["comparisons"][key] = {
446
+ "both_vs_confidence_metric_delta_percentage_points": 100 * (learned["test_metric"] - confidence["test_metric"]),
447
+ "both_vs_confidence_call_delta_percentage_points": 100 * (learned["actual_test_call_fraction"] - confidence["actual_test_call_fraction"]),
448
+ "both_vs_confidence_flops_relative_percent": 100 * (learned["flops_gflops_per_input"] / confidence["flops_gflops_per_input"] - 1),
449
+ "both_vs_confidence_latency_relative_percent": 100 * (learned["latency_ms_per_input"] / confidence["latency_ms_per_input"] - 1),
450
+ }
451
+ write_json(root / "cost_summary.json", summary)
452
+ (root / "成本性能补充报告.md").write_text(report_markdown(components, all_selected, all_fixed, root, summary["comparisons"]), encoding="utf-8")
453
+ print(json.dumps({"status": "complete", "output_dir": str(root), "curve_rows": len(all_curves), "fixed_rows": len(all_fixed)}, ensure_ascii=False))
454
+
455
+
456
+ if __name__ == "__main__":
457
+ main()
releases/20260913_lr/paper_data/cost_20260911/minicpm_grefcoco.json ADDED
@@ -0,0 +1,385 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "complete",
3
+ "model_kind": "MiniCPM",
4
+ "task": "grefcoco",
5
+ "expert": "instancevg",
6
+ "model_path": "/mnt/HithinkOmniSSD/user_workspace/caisihang/checkpoints/MiniCPM-V-4_5",
7
+ "device": "cuda:0",
8
+ "gpu_memory_utilization": 0.75,
9
+ "sample_count_requested": 32,
10
+ "warmup_count": 2,
11
+ "measured_count": 30,
12
+ "batch_size": 4,
13
+ "sample_ids": [
14
+ "testA-24562-67519",
15
+ "testB-37093-102233",
16
+ "testB-23966-65871",
17
+ "testB-66119-182290",
18
+ "testA-81450-224192",
19
+ "testB-33122-91196",
20
+ "testB-80267-220980",
21
+ "testA-47140-129791",
22
+ "testA-24439-67195",
23
+ "testA-54310-149436",
24
+ "testA-79491-218834",
25
+ "testA-10078-27741",
26
+ "testB-57040-156982",
27
+ "testA-26858-73835",
28
+ "testB-44430-122263",
29
+ "testA-60760-167399",
30
+ "testB-3182-8806",
31
+ "testB-947-2650",
32
+ "testA-73301-202004",
33
+ "testA-29200-80379",
34
+ "testB-42734-117684",
35
+ "testB-46676-128533",
36
+ "testB-8468-23291",
37
+ "testA-54557-150151",
38
+ "testA-58479-160960",
39
+ "testA-81251-223648",
40
+ "testA-5038-13944",
41
+ "testB-37760-104089",
42
+ "testA-62943-173467",
43
+ "testB-12467-34323",
44
+ "testA-80534-221720",
45
+ "testB-53121-146261"
46
+ ],
47
+ "batches": [
48
+ {
49
+ "sample_ids": [
50
+ "testB-23966-65871",
51
+ "testB-66119-182290",
52
+ "testA-81450-224192",
53
+ "testB-33122-91196"
54
+ ],
55
+ "batch_size": 4,
56
+ "generation_latency_seconds": 6.051569456933066,
57
+ "batch_wall_latency_seconds": 6.090657560969703,
58
+ "amortized_generation_ms_per_sample": 1512.8923642332666,
59
+ "amortized_batch_wall_ms_per_sample": 1522.6643902424257,
60
+ "outputs": [
61
+ {
62
+ "finish_reason": "length",
63
+ "stop_reason": null,
64
+ "prompt_tokens": 528,
65
+ "completion_tokens": 512,
66
+ "text_characters": 528
67
+ },
68
+ {
69
+ "finish_reason": "stop",
70
+ "stop_reason": null,
71
+ "prompt_tokens": 526,
72
+ "completion_tokens": 87,
73
+ "text_characters": 140
74
+ },
75
+ {
76
+ "finish_reason": "length",
77
+ "stop_reason": null,
78
+ "prompt_tokens": 523,
79
+ "completion_tokens": 512,
80
+ "text_characters": 528
81
+ },
82
+ {
83
+ "finish_reason": "length",
84
+ "stop_reason": null,
85
+ "prompt_tokens": 527,
86
+ "completion_tokens": 512,
87
+ "text_characters": 528
88
+ }
89
+ ]
90
+ },
91
+ {
92
+ "sample_ids": [
93
+ "testB-80267-220980",
94
+ "testA-47140-129791",
95
+ "testA-24439-67195",
96
+ "testA-54310-149436"
97
+ ],
98
+ "batch_size": 4,
99
+ "generation_latency_seconds": 6.01708423695527,
100
+ "batch_wall_latency_seconds": 6.064165806979872,
101
+ "amortized_generation_ms_per_sample": 1504.2710592388175,
102
+ "amortized_batch_wall_ms_per_sample": 1516.041451744968,
103
+ "outputs": [
104
+ {
105
+ "finish_reason": "stop",
106
+ "stop_reason": null,
107
+ "prompt_tokens": 525,
108
+ "completion_tokens": 5,
109
+ "text_characters": 14
110
+ },
111
+ {
112
+ "finish_reason": "stop",
113
+ "stop_reason": null,
114
+ "prompt_tokens": 524,
115
+ "completion_tokens": 29,
116
+ "text_characters": 52
117
+ },
118
+ {
119
+ "finish_reason": "stop",
120
+ "stop_reason": null,
121
+ "prompt_tokens": 524,
122
+ "completion_tokens": 33,
123
+ "text_characters": 56
124
+ },
125
+ {
126
+ "finish_reason": "length",
127
+ "stop_reason": null,
128
+ "prompt_tokens": 522,
129
+ "completion_tokens": 512,
130
+ "text_characters": 528
131
+ }
132
+ ]
133
+ },
134
+ {
135
+ "sample_ids": [
136
+ "testA-79491-218834",
137
+ "testA-10078-27741",
138
+ "testB-57040-156982",
139
+ "testA-26858-73835"
140
+ ],
141
+ "batch_size": 4,
142
+ "generation_latency_seconds": 6.010764242964797,
143
+ "batch_wall_latency_seconds": 6.046397175989114,
144
+ "amortized_generation_ms_per_sample": 1502.6910607411992,
145
+ "amortized_batch_wall_ms_per_sample": 1511.5992939972784,
146
+ "outputs": [
147
+ {
148
+ "finish_reason": "stop",
149
+ "stop_reason": null,
150
+ "prompt_tokens": 524,
151
+ "completion_tokens": 62,
152
+ "text_characters": 100
153
+ },
154
+ {
155
+ "finish_reason": "stop",
156
+ "stop_reason": null,
157
+ "prompt_tokens": 529,
158
+ "completion_tokens": 87,
159
+ "text_characters": 140
160
+ },
161
+ {
162
+ "finish_reason": "length",
163
+ "stop_reason": null,
164
+ "prompt_tokens": 528,
165
+ "completion_tokens": 512,
166
+ "text_characters": 1690
167
+ },
168
+ {
169
+ "finish_reason": "stop",
170
+ "stop_reason": null,
171
+ "prompt_tokens": 521,
172
+ "completion_tokens": 5,
173
+ "text_characters": 14
174
+ }
175
+ ]
176
+ },
177
+ {
178
+ "sample_ids": [
179
+ "testB-44430-122263",
180
+ "testA-60760-167399",
181
+ "testB-3182-8806",
182
+ "testB-947-2650"
183
+ ],
184
+ "batch_size": 4,
185
+ "generation_latency_seconds": 2.593448459985666,
186
+ "batch_wall_latency_seconds": 2.6335477660177276,
187
+ "amortized_generation_ms_per_sample": 648.3621149964165,
188
+ "amortized_batch_wall_ms_per_sample": 658.3869415044319,
189
+ "outputs": [
190
+ {
191
+ "finish_reason": "stop",
192
+ "stop_reason": null,
193
+ "prompt_tokens": 525,
194
+ "completion_tokens": 6,
195
+ "text_characters": 14
196
+ },
197
+ {
198
+ "finish_reason": "stop",
199
+ "stop_reason": null,
200
+ "prompt_tokens": 392,
201
+ "completion_tokens": 33,
202
+ "text_characters": 56
203
+ },
204
+ {
205
+ "finish_reason": "stop",
206
+ "stop_reason": null,
207
+ "prompt_tokens": 392,
208
+ "completion_tokens": 33,
209
+ "text_characters": 56
210
+ },
211
+ {
212
+ "finish_reason": "stop",
213
+ "stop_reason": null,
214
+ "prompt_tokens": 528,
215
+ "completion_tokens": 208,
216
+ "text_characters": 321
217
+ }
218
+ ]
219
+ },
220
+ {
221
+ "sample_ids": [
222
+ "testA-73301-202004",
223
+ "testA-29200-80379",
224
+ "testB-42734-117684",
225
+ "testB-46676-128533"
226
+ ],
227
+ "batch_size": 4,
228
+ "generation_latency_seconds": 6.017314173979685,
229
+ "batch_wall_latency_seconds": 6.065163187915459,
230
+ "amortized_generation_ms_per_sample": 1504.3285434949212,
231
+ "amortized_batch_wall_ms_per_sample": 1516.2907969788648,
232
+ "outputs": [
233
+ {
234
+ "finish_reason": "stop",
235
+ "stop_reason": null,
236
+ "prompt_tokens": 525,
237
+ "completion_tokens": 61,
238
+ "text_characters": 99
239
+ },
240
+ {
241
+ "finish_reason": "stop",
242
+ "stop_reason": null,
243
+ "prompt_tokens": 520,
244
+ "completion_tokens": 5,
245
+ "text_characters": 14
246
+ },
247
+ {
248
+ "finish_reason": "length",
249
+ "stop_reason": null,
250
+ "prompt_tokens": 523,
251
+ "completion_tokens": 512,
252
+ "text_characters": 528
253
+ },
254
+ {
255
+ "finish_reason": "stop",
256
+ "stop_reason": null,
257
+ "prompt_tokens": 528,
258
+ "completion_tokens": 5,
259
+ "text_characters": 14
260
+ }
261
+ ]
262
+ },
263
+ {
264
+ "sample_ids": [
265
+ "testB-8468-23291",
266
+ "testA-54557-150151",
267
+ "testA-58479-160960",
268
+ "testA-81251-223648"
269
+ ],
270
+ "batch_size": 4,
271
+ "generation_latency_seconds": 1.6802827849751338,
272
+ "batch_wall_latency_seconds": 1.716626849025488,
273
+ "amortized_generation_ms_per_sample": 420.07069624378346,
274
+ "amortized_batch_wall_ms_per_sample": 429.156712256372,
275
+ "outputs": [
276
+ {
277
+ "finish_reason": "stop",
278
+ "stop_reason": null,
279
+ "prompt_tokens": 539,
280
+ "completion_tokens": 62,
281
+ "text_characters": 100
282
+ },
283
+ {
284
+ "finish_reason": "stop",
285
+ "stop_reason": null,
286
+ "prompt_tokens": 522,
287
+ "completion_tokens": 30,
288
+ "text_characters": 53
289
+ },
290
+ {
291
+ "finish_reason": "stop",
292
+ "stop_reason": null,
293
+ "prompt_tokens": 529,
294
+ "completion_tokens": 6,
295
+ "text_characters": 14
296
+ },
297
+ {
298
+ "finish_reason": "stop",
299
+ "stop_reason": null,
300
+ "prompt_tokens": 525,
301
+ "completion_tokens": 122,
302
+ "text_characters": 190
303
+ }
304
+ ]
305
+ },
306
+ {
307
+ "sample_ids": [
308
+ "testA-5038-13944",
309
+ "testB-37760-104089",
310
+ "testA-62943-173467",
311
+ "testB-12467-34323"
312
+ ],
313
+ "batch_size": 4,
314
+ "generation_latency_seconds": 5.998426596052013,
315
+ "batch_wall_latency_seconds": 6.038230134989135,
316
+ "amortized_generation_ms_per_sample": 1499.6066490130033,
317
+ "amortized_batch_wall_ms_per_sample": 1509.5575337472837,
318
+ "outputs": [
319
+ {
320
+ "finish_reason": "stop",
321
+ "stop_reason": null,
322
+ "prompt_tokens": 522,
323
+ "completion_tokens": 39,
324
+ "text_characters": 64
325
+ },
326
+ {
327
+ "finish_reason": "length",
328
+ "stop_reason": null,
329
+ "prompt_tokens": 536,
330
+ "completion_tokens": 512,
331
+ "text_characters": 528
332
+ },
333
+ {
334
+ "finish_reason": "stop",
335
+ "stop_reason": null,
336
+ "prompt_tokens": 538,
337
+ "completion_tokens": 33,
338
+ "text_characters": 56
339
+ },
340
+ {
341
+ "finish_reason": "stop",
342
+ "stop_reason": null,
343
+ "prompt_tokens": 522,
344
+ "completion_tokens": 5,
345
+ "text_characters": 14
346
+ }
347
+ ]
348
+ },
349
+ {
350
+ "sample_ids": [
351
+ "testA-80534-221720",
352
+ "testB-53121-146261"
353
+ ],
354
+ "batch_size": 2,
355
+ "generation_latency_seconds": 5.806203712942079,
356
+ "batch_wall_latency_seconds": 5.831518966006115,
357
+ "amortized_generation_ms_per_sample": 2903.1018564710394,
358
+ "amortized_batch_wall_ms_per_sample": 2915.7594830030575,
359
+ "outputs": [
360
+ {
361
+ "finish_reason": "length",
362
+ "stop_reason": null,
363
+ "prompt_tokens": 390,
364
+ "completion_tokens": 512,
365
+ "text_characters": 528
366
+ },
367
+ {
368
+ "finish_reason": "stop",
369
+ "stop_reason": null,
370
+ "prompt_tokens": 523,
371
+ "completion_tokens": 33,
372
+ "text_characters": 56
373
+ }
374
+ ]
375
+ }
376
+ ],
377
+ "mean_generation_ms_per_sample": 1436.915543054056,
378
+ "median_generation_ms_per_sample": 1503.4810599900084,
379
+ "mean_batch_wall_ms_per_sample": 1447.4320754343353,
380
+ "median_batch_wall_ms_per_sample": 1513.8203728711233,
381
+ "mean_prompt_tokens": 512.6666666666666,
382
+ "mean_completion_tokens": 169.5,
383
+ "latency_scope": "A800 vLLM-0.18.0 native generation latency with the frozen task prompt and structured-output schema; model loading and warmup excluded. Batch wall time includes the real request/image preparation path.",
384
+ "flops_scope": "This file measures latency only. Counted FLOPs are measured separately by the HF reference probe and are not claimed to be complete vLLM kernel FLOPs."
385
+ }
releases/20260913_lr/paper_data/cost_20260911/minicpm_grefcoco_flops.json ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "complete",
3
+ "model_kind": "MiniCPM",
4
+ "task": "grefcoco",
5
+ "model_path": "/mnt/HithinkOmniSSD/user_workspace/caisihang/checkpoints/MiniCPM-V-4_5",
6
+ "sample_count": 4,
7
+ "records": [
8
+ {
9
+ "sample_id": "testB-37890-104442",
10
+ "prompt_tokens_in_reference_tensor": 528,
11
+ "input_shapes": {
12
+ "input_ids": [
13
+ 1,
14
+ 528
15
+ ],
16
+ "attention_mask": [
17
+ 1,
18
+ 528
19
+ ]
20
+ },
21
+ "prefill_flops_counted": 7500407080960,
22
+ "decode_one_token_flops_counted": 15446671488,
23
+ "prefill_gflops_counted": 7500.40708096,
24
+ "decode_one_token_gflops_counted": 15.446671488,
25
+ "prefill_operators": {
26
+ "aten.bmm": 164433561600,
27
+ "aten.mm": 7335973519360
28
+ },
29
+ "decode_operators": {
30
+ "aten.bmm": 312017024,
31
+ "aten.mm": 15134654464
32
+ }
33
+ },
34
+ {
35
+ "sample_id": "testA-73305-202016",
36
+ "prompt_tokens_in_reference_tensor": 525,
37
+ "input_shapes": {
38
+ "input_ids": [
39
+ 1,
40
+ 525
41
+ ],
42
+ "attention_mask": [
43
+ 1,
44
+ 525
45
+ ]
46
+ },
47
+ "prefill_flops_counted": 7456869222016,
48
+ "decode_one_token_flops_counted": 15444902016,
49
+ "prefill_gflops_counted": 7456.869222016,
50
+ "decode_one_token_gflops_counted": 15.444902016,
51
+ "prefill_operators": {
52
+ "aten.bmm": 162570307200,
53
+ "aten.mm": 7294298914816
54
+ },
55
+ "decode_operators": {
56
+ "aten.bmm": 310247552,
57
+ "aten.mm": 15134654464
58
+ }
59
+ },
60
+ {
61
+ "sample_id": "testB-24036-66087",
62
+ "prompt_tokens_in_reference_tensor": 522,
63
+ "input_shapes": {
64
+ "input_ids": [
65
+ 1,
66
+ 522
67
+ ],
68
+ "attention_mask": [
69
+ 1,
70
+ 522
71
+ ]
72
+ },
73
+ "prefill_flops_counted": 7413341979904,
74
+ "decode_one_token_flops_counted": 15443132544,
75
+ "prefill_gflops_counted": 7413.341979904,
76
+ "decode_one_token_gflops_counted": 15.443132544,
77
+ "prefill_operators": {
78
+ "aten.bmm": 160717669632,
79
+ "aten.mm": 7252624310272
80
+ },
81
+ "decode_operators": {
82
+ "aten.bmm": 308478080,
83
+ "aten.mm": 15134654464
84
+ }
85
+ },
86
+ {
87
+ "sample_id": "testA-5038-13945",
88
+ "prompt_tokens_in_reference_tensor": 528,
89
+ "input_shapes": {
90
+ "input_ids": [
91
+ 1,
92
+ 528
93
+ ],
94
+ "attention_mask": [
95
+ 1,
96
+ 528
97
+ ]
98
+ },
99
+ "prefill_flops_counted": 7500407080960,
100
+ "decode_one_token_flops_counted": 15446671488,
101
+ "prefill_gflops_counted": 7500.40708096,
102
+ "decode_one_token_gflops_counted": 15.446671488,
103
+ "prefill_operators": {
104
+ "aten.bmm": 164433561600,
105
+ "aten.mm": 7335973519360
106
+ },
107
+ "decode_operators": {
108
+ "aten.bmm": 312017024,
109
+ "aten.mm": 15134654464
110
+ }
111
+ }
112
+ ],
113
+ "mean_prefill_gflops_counted": 7467.756340960001,
114
+ "mean_decode_one_token_gflops_counted": 15.445344384,
115
+ "scope": "HF reference execution on the A800 with real task inputs; dense products/convolutions/supported attention counted with multiply-add=2 FLOPs. vLLM custom kernels, elementwise/normalization/nonlinearities and other unsupported operations are not complete.",
116
+ "answer_generation": "not performed; one known-token decode probe only"
117
+ }
releases/20260913_lr/paper_data/cost_20260911/qwen_construction.json ADDED
@@ -0,0 +1,385 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "complete",
3
+ "model_kind": "Qwen3.8",
4
+ "task": "construction",
5
+ "expert": "yolo26x",
6
+ "model_path": "/mnt/HithinkOmniSSD/user_workspace/caisihang/checkpoints/Qwen3.8-27B-FP8",
7
+ "device": "cuda:0",
8
+ "gpu_memory_utilization": 0.75,
9
+ "sample_count_requested": 32,
10
+ "warmup_count": 2,
11
+ "measured_count": 30,
12
+ "batch_size": 4,
13
+ "sample_ids": [
14
+ "0000772",
15
+ "0003400",
16
+ "0003077",
17
+ "0004213",
18
+ "0002162",
19
+ "0003303",
20
+ "0004535",
21
+ "0001299",
22
+ "0000762",
23
+ "0001453",
24
+ "0002112",
25
+ "0000382",
26
+ "0003936",
27
+ "0000833",
28
+ "0003551",
29
+ "0001683",
30
+ "0002571",
31
+ "0002523",
32
+ "0001939",
33
+ "0000904",
34
+ "0003516",
35
+ "0003633",
36
+ "0002690",
37
+ "0001464",
38
+ "0001595",
39
+ "0002168",
40
+ "0000183",
41
+ "0003404",
42
+ "0001727",
43
+ "0002796",
44
+ "0002153",
45
+ "0003848"
46
+ ],
47
+ "batches": [
48
+ {
49
+ "sample_ids": [
50
+ "0003077",
51
+ "0004213",
52
+ "0002162",
53
+ "0003303"
54
+ ],
55
+ "batch_size": 4,
56
+ "generation_latency_seconds": 4.881686039967462,
57
+ "batch_wall_latency_seconds": 4.988335099071264,
58
+ "amortized_generation_ms_per_sample": 1220.4215099918656,
59
+ "amortized_batch_wall_ms_per_sample": 1247.083774767816,
60
+ "outputs": [
61
+ {
62
+ "finish_reason": "stop",
63
+ "stop_reason": null,
64
+ "prompt_tokens": 1705,
65
+ "completion_tokens": 110,
66
+ "text_characters": 345
67
+ },
68
+ {
69
+ "finish_reason": "stop",
70
+ "stop_reason": null,
71
+ "prompt_tokens": 1705,
72
+ "completion_tokens": 110,
73
+ "text_characters": 345
74
+ },
75
+ {
76
+ "finish_reason": "stop",
77
+ "stop_reason": null,
78
+ "prompt_tokens": 1705,
79
+ "completion_tokens": 110,
80
+ "text_characters": 345
81
+ },
82
+ {
83
+ "finish_reason": "stop",
84
+ "stop_reason": null,
85
+ "prompt_tokens": 1705,
86
+ "completion_tokens": 110,
87
+ "text_characters": 345
88
+ }
89
+ ]
90
+ },
91
+ {
92
+ "sample_ids": [
93
+ "0004535",
94
+ "0001299",
95
+ "0000762",
96
+ "0001453"
97
+ ],
98
+ "batch_size": 4,
99
+ "generation_latency_seconds": 7.0890667490893975,
100
+ "batch_wall_latency_seconds": 7.216836072970182,
101
+ "amortized_generation_ms_per_sample": 1772.2666872723494,
102
+ "amortized_batch_wall_ms_per_sample": 1804.2090182425454,
103
+ "outputs": [
104
+ {
105
+ "finish_reason": "stop",
106
+ "stop_reason": null,
107
+ "prompt_tokens": 1705,
108
+ "completion_tokens": 110,
109
+ "text_characters": 345
110
+ },
111
+ {
112
+ "finish_reason": "stop",
113
+ "stop_reason": null,
114
+ "prompt_tokens": 1694,
115
+ "completion_tokens": 110,
116
+ "text_characters": 345
117
+ },
118
+ {
119
+ "finish_reason": "stop",
120
+ "stop_reason": null,
121
+ "prompt_tokens": 1531,
122
+ "completion_tokens": 144,
123
+ "text_characters": 435
124
+ },
125
+ {
126
+ "finish_reason": "stop",
127
+ "stop_reason": null,
128
+ "prompt_tokens": 1531,
129
+ "completion_tokens": 226,
130
+ "text_characters": 616
131
+ }
132
+ ]
133
+ },
134
+ {
135
+ "sample_ids": [
136
+ "0002112",
137
+ "0000382",
138
+ "0003936",
139
+ "0000833"
140
+ ],
141
+ "batch_size": 4,
142
+ "generation_latency_seconds": 5.4999597480054945,
143
+ "batch_wall_latency_seconds": 5.588348102988675,
144
+ "amortized_generation_ms_per_sample": 1374.9899370013736,
145
+ "amortized_batch_wall_ms_per_sample": 1397.0870257471688,
146
+ "outputs": [
147
+ {
148
+ "finish_reason": "stop",
149
+ "stop_reason": null,
150
+ "prompt_tokens": 1405,
151
+ "completion_tokens": 153,
152
+ "text_characters": 480
153
+ },
154
+ {
155
+ "finish_reason": "stop",
156
+ "stop_reason": null,
157
+ "prompt_tokens": 1705,
158
+ "completion_tokens": 110,
159
+ "text_characters": 345
160
+ },
161
+ {
162
+ "finish_reason": "stop",
163
+ "stop_reason": null,
164
+ "prompt_tokens": 1531,
165
+ "completion_tokens": 110,
166
+ "text_characters": 345
167
+ },
168
+ {
169
+ "finish_reason": "stop",
170
+ "stop_reason": null,
171
+ "prompt_tokens": 1683,
172
+ "completion_tokens": 139,
173
+ "text_characters": 409
174
+ }
175
+ ]
176
+ },
177
+ {
178
+ "sample_ids": [
179
+ "0003551",
180
+ "0001683",
181
+ "0002571",
182
+ "0002523"
183
+ ],
184
+ "batch_size": 4,
185
+ "generation_latency_seconds": 4.797023626044393,
186
+ "batch_wall_latency_seconds": 4.88577020005323,
187
+ "amortized_generation_ms_per_sample": 1199.2559065110981,
188
+ "amortized_batch_wall_ms_per_sample": 1221.4425500133075,
189
+ "outputs": [
190
+ {
191
+ "finish_reason": "stop",
192
+ "stop_reason": null,
193
+ "prompt_tokens": 1705,
194
+ "completion_tokens": 110,
195
+ "text_characters": 345
196
+ },
197
+ {
198
+ "finish_reason": "stop",
199
+ "stop_reason": null,
200
+ "prompt_tokens": 1705,
201
+ "completion_tokens": 110,
202
+ "text_characters": 345
203
+ },
204
+ {
205
+ "finish_reason": "stop",
206
+ "stop_reason": null,
207
+ "prompt_tokens": 1694,
208
+ "completion_tokens": 110,
209
+ "text_characters": 345
210
+ },
211
+ {
212
+ "finish_reason": "stop",
213
+ "stop_reason": null,
214
+ "prompt_tokens": 1705,
215
+ "completion_tokens": 110,
216
+ "text_characters": 345
217
+ }
218
+ ]
219
+ },
220
+ {
221
+ "sample_ids": [
222
+ "0001939",
223
+ "0000904",
224
+ "0003516",
225
+ "0003633"
226
+ ],
227
+ "batch_size": 4,
228
+ "generation_latency_seconds": 6.3565617019776255,
229
+ "batch_wall_latency_seconds": 6.429374794010073,
230
+ "amortized_generation_ms_per_sample": 1589.1404254944064,
231
+ "amortized_batch_wall_ms_per_sample": 1607.3436985025182,
232
+ "outputs": [
233
+ {
234
+ "finish_reason": "stop",
235
+ "stop_reason": null,
236
+ "prompt_tokens": 1705,
237
+ "completion_tokens": 110,
238
+ "text_characters": 345
239
+ },
240
+ {
241
+ "finish_reason": "stop",
242
+ "stop_reason": null,
243
+ "prompt_tokens": 1705,
244
+ "completion_tokens": 147,
245
+ "text_characters": 465
246
+ },
247
+ {
248
+ "finish_reason": "stop",
249
+ "stop_reason": null,
250
+ "prompt_tokens": 1683,
251
+ "completion_tokens": 185,
252
+ "text_characters": 574
253
+ },
254
+ {
255
+ "finish_reason": "stop",
256
+ "stop_reason": null,
257
+ "prompt_tokens": 1405,
258
+ "completion_tokens": 141,
259
+ "text_characters": 357
260
+ }
261
+ ]
262
+ },
263
+ {
264
+ "sample_ids": [
265
+ "0002690",
266
+ "0001464",
267
+ "0001595",
268
+ "0002168"
269
+ ],
270
+ "batch_size": 4,
271
+ "generation_latency_seconds": 11.480005273944698,
272
+ "batch_wall_latency_seconds": 11.721151808975264,
273
+ "amortized_generation_ms_per_sample": 2870.0013184861746,
274
+ "amortized_batch_wall_ms_per_sample": 2930.287952243816,
275
+ "outputs": [
276
+ {
277
+ "finish_reason": "stop",
278
+ "stop_reason": null,
279
+ "prompt_tokens": 1705,
280
+ "completion_tokens": 110,
281
+ "text_characters": 345
282
+ },
283
+ {
284
+ "finish_reason": "stop",
285
+ "stop_reason": null,
286
+ "prompt_tokens": 1705,
287
+ "completion_tokens": 110,
288
+ "text_characters": 345
289
+ },
290
+ {
291
+ "finish_reason": "stop",
292
+ "stop_reason": null,
293
+ "prompt_tokens": 1705,
294
+ "completion_tokens": 177,
295
+ "text_characters": 545
296
+ },
297
+ {
298
+ "finish_reason": "stop",
299
+ "stop_reason": null,
300
+ "prompt_tokens": 1705,
301
+ "completion_tokens": 155,
302
+ "text_characters": 481
303
+ }
304
+ ]
305
+ },
306
+ {
307
+ "sample_ids": [
308
+ "0000183",
309
+ "0003404",
310
+ "0001727",
311
+ "0002796"
312
+ ],
313
+ "batch_size": 4,
314
+ "generation_latency_seconds": 13.841306244954467,
315
+ "batch_wall_latency_seconds": 13.994719217997044,
316
+ "amortized_generation_ms_per_sample": 3460.3265612386167,
317
+ "amortized_batch_wall_ms_per_sample": 3498.679804499261,
318
+ "outputs": [
319
+ {
320
+ "finish_reason": "stop",
321
+ "stop_reason": null,
322
+ "prompt_tokens": 1705,
323
+ "completion_tokens": 157,
324
+ "text_characters": 510
325
+ },
326
+ {
327
+ "finish_reason": "stop",
328
+ "stop_reason": null,
329
+ "prompt_tokens": 1705,
330
+ "completion_tokens": 150,
331
+ "text_characters": 462
332
+ },
333
+ {
334
+ "finish_reason": "stop",
335
+ "stop_reason": null,
336
+ "prompt_tokens": 1531,
337
+ "completion_tokens": 169,
338
+ "text_characters": 508
339
+ },
340
+ {
341
+ "finish_reason": "stop",
342
+ "stop_reason": null,
343
+ "prompt_tokens": 1705,
344
+ "completion_tokens": 187,
345
+ "text_characters": 511
346
+ }
347
+ ]
348
+ },
349
+ {
350
+ "sample_ids": [
351
+ "0002153",
352
+ "0003848"
353
+ ],
354
+ "batch_size": 2,
355
+ "generation_latency_seconds": 14.003285999991931,
356
+ "batch_wall_latency_seconds": 14.185666976962239,
357
+ "amortized_generation_ms_per_sample": 7001.6429999959655,
358
+ "amortized_batch_wall_ms_per_sample": 7092.833488481119,
359
+ "outputs": [
360
+ {
361
+ "finish_reason": "stop",
362
+ "stop_reason": null,
363
+ "prompt_tokens": 1705,
364
+ "completion_tokens": 221,
365
+ "text_characters": 616
366
+ },
367
+ {
368
+ "finish_reason": "stop",
369
+ "stop_reason": null,
370
+ "prompt_tokens": 1705,
371
+ "completion_tokens": 110,
372
+ "text_characters": 345
373
+ }
374
+ ]
375
+ }
376
+ ],
377
+ "mean_generation_ms_per_sample": 2561.0056682489812,
378
+ "median_generation_ms_per_sample": 1680.7035563833779,
379
+ "mean_batch_wall_ms_per_sample": 2599.870914062194,
380
+ "median_batch_wall_ms_per_sample": 1705.7763583725318,
381
+ "mean_prompt_tokens": 1659.6,
382
+ "mean_completion_tokens": 137.03333333333333,
383
+ "latency_scope": "A800 vLLM-0.18.0 native generation latency with the frozen task prompt and structured-output schema; model loading and warmup excluded. Batch wall time includes the real request/image preparation path.",
384
+ "flops_scope": "This file measures latency only. Counted FLOPs are measured separately by the HF reference probe and are not claimed to be complete vLLM kernel FLOPs."
385
+ }
releases/20260913_lr/paper_data/cost_20260911/qwen_construction_flops.json ADDED
@@ -0,0 +1,177 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "complete",
3
+ "model_kind": "Qwen3.8",
4
+ "task": "construction",
5
+ "model_path": "/mnt/HithinkOmniSSD/user_workspace/caisihang/checkpoints/Qwen3.8-27B-FP8",
6
+ "sample_count": 4,
7
+ "records": [
8
+ {
9
+ "sample_id": "0003439",
10
+ "prompt_tokens_in_reference_tensor": 1705,
11
+ "input_shapes": {
12
+ "input_ids": [
13
+ 1,
14
+ 1705
15
+ ],
16
+ "attention_mask": [
17
+ 1,
18
+ 1705
19
+ ],
20
+ "mm_token_type_ids": [
21
+ 1,
22
+ 1705
23
+ ],
24
+ "pixel_values": [
25
+ 3888,
26
+ 1536
27
+ ],
28
+ "image_grid_thw": [
29
+ 1,
30
+ 3
31
+ ]
32
+ },
33
+ "prefill_flops_counted": 90083398713024,
34
+ "decode_one_token_flops_counted": 51922600128,
35
+ "prefill_gflops_counted": 90083.398713024,
36
+ "decode_one_token_gflops_counted": 51.922600128,
37
+ "prefill_operators": {
38
+ "aten.convolution": 20475543552,
39
+ "aten.addmm": 3283626885120,
40
+ "aten.bmm": 3741356981952,
41
+ "aten.mm": 83037939302400
42
+ },
43
+ "decode_operators": {
44
+ "aten.bmm": 670826688,
45
+ "aten.mm": 51243909120,
46
+ "aten.convolution": 7864320
47
+ }
48
+ },
49
+ {
50
+ "sample_id": "0001957",
51
+ "prompt_tokens_in_reference_tensor": 1705,
52
+ "input_shapes": {
53
+ "input_ids": [
54
+ 1,
55
+ 1705
56
+ ],
57
+ "attention_mask": [
58
+ 1,
59
+ 1705
60
+ ],
61
+ "mm_token_type_ids": [
62
+ 1,
63
+ 1705
64
+ ],
65
+ "pixel_values": [
66
+ 3888,
67
+ 1536
68
+ ],
69
+ "image_grid_thw": [
70
+ 1,
71
+ 3
72
+ ]
73
+ },
74
+ "prefill_flops_counted": 90083398713024,
75
+ "decode_one_token_flops_counted": 51922600128,
76
+ "prefill_gflops_counted": 90083.398713024,
77
+ "decode_one_token_gflops_counted": 51.922600128,
78
+ "prefill_operators": {
79
+ "aten.convolution": 20475543552,
80
+ "aten.addmm": 3283626885120,
81
+ "aten.bmm": 3741356981952,
82
+ "aten.mm": 83037939302400
83
+ },
84
+ "decode_operators": {
85
+ "aten.bmm": 670826688,
86
+ "aten.mm": 51243909120,
87
+ "aten.convolution": 7864320
88
+ }
89
+ },
90
+ {
91
+ "sample_id": "0003113",
92
+ "prompt_tokens_in_reference_tensor": 1531,
93
+ "input_shapes": {
94
+ "input_ids": [
95
+ 1,
96
+ 1531
97
+ ],
98
+ "attention_mask": [
99
+ 1,
100
+ 1531
101
+ ],
102
+ "mm_token_type_ids": [
103
+ 1,
104
+ 1531
105
+ ],
106
+ "pixel_values": [
107
+ 3192,
108
+ 1536
109
+ ],
110
+ "image_grid_thw": [
111
+ 1,
112
+ 3
113
+ ]
114
+ },
115
+ "prefill_flops_counted": 80104234744896,
116
+ "decode_one_token_flops_counted": 51854180544,
117
+ "prefill_gflops_counted": 80104.234744896,
118
+ "decode_one_token_gflops_counted": 51.854180544,
119
+ "prefill_operators": {
120
+ "aten.convolution": 17328242688,
121
+ "aten.addmm": 2695817134080,
122
+ "aten.bmm": 2827143609408,
123
+ "aten.mm": 74563945758720
124
+ },
125
+ "decode_operators": {
126
+ "aten.bmm": 602407104,
127
+ "aten.mm": 51243909120,
128
+ "aten.convolution": 7864320
129
+ }
130
+ },
131
+ {
132
+ "sample_id": "0000184",
133
+ "prompt_tokens_in_reference_tensor": 1705,
134
+ "input_shapes": {
135
+ "input_ids": [
136
+ 1,
137
+ 1705
138
+ ],
139
+ "attention_mask": [
140
+ 1,
141
+ 1705
142
+ ],
143
+ "mm_token_type_ids": [
144
+ 1,
145
+ 1705
146
+ ],
147
+ "pixel_values": [
148
+ 3888,
149
+ 1536
150
+ ],
151
+ "image_grid_thw": [
152
+ 1,
153
+ 3
154
+ ]
155
+ },
156
+ "prefill_flops_counted": 90083398713024,
157
+ "decode_one_token_flops_counted": 51922600128,
158
+ "prefill_gflops_counted": 90083.398713024,
159
+ "decode_one_token_gflops_counted": 51.922600128,
160
+ "prefill_operators": {
161
+ "aten.convolution": 20475543552,
162
+ "aten.addmm": 3283626885120,
163
+ "aten.bmm": 3741356981952,
164
+ "aten.mm": 83037939302400
165
+ },
166
+ "decode_operators": {
167
+ "aten.bmm": 670826688,
168
+ "aten.mm": 51243909120,
169
+ "aten.convolution": 7864320
170
+ }
171
+ }
172
+ ],
173
+ "mean_prefill_gflops_counted": 87588.607720992,
174
+ "mean_decode_one_token_gflops_counted": 51.905495232,
175
+ "scope": "HF reference execution on the A800 with real task inputs; dense products/convolutions/supported attention counted with multiply-add=2 FLOPs. vLLM custom kernels, elementwise/normalization/nonlinearities and other unsupported operations are not complete.",
176
+ "answer_generation": "not performed; one known-token decode probe only"
177
+ }
releases/20260913_lr/paper_data/cost_20260911/router_architecture.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "pair_key": "grefcoco|instancevg|MiniCPM",
4
+ "task": "grefcoco",
5
+ "expert": "instancevg",
6
+ "endpoint": "MiniCPM",
7
+ "branches": ["image", "text", "hidden", "confidence", "output"],
8
+ "input_dimensions": {
9
+ "image_flat_plus_valid": 1538,
10
+ "text": 768,
11
+ "hidden": 768,
12
+ "confidence": 8,
13
+ "output": 2
14
+ },
15
+ "representation_dim": 128,
16
+ "head_hidden_dim": 128,
17
+ "class_count": 4,
18
+ "source_checkpoint": "/mnt/HithinkOmniSSD/user_workspace/caisihang/research/ICASSP_ModelCollaboration/outputs/router_method_improvement_20260911_r1/conditions/both_instancevg_MiniCPM_seed42/best.pt"
19
+ },
20
+ {
21
+ "pair_key": "construction|yolo26x|Qwen3.8",
22
+ "task": "construction",
23
+ "expert": "yolo26x",
24
+ "endpoint": "Qwen3.8",
25
+ "branches": ["image", "hidden", "confidence", "output"],
26
+ "input_dimensions": {
27
+ "image_flat_plus_valid": 1538,
28
+ "hidden": 768,
29
+ "confidence": 8,
30
+ "output": 8
31
+ },
32
+ "representation_dim": 128,
33
+ "head_hidden_dim": 128,
34
+ "class_count": 4,
35
+ "source_checkpoint": "/mnt/HithinkOmniSSD/user_workspace/caisihang/research/ICASSP_ModelCollaboration/outputs/router_method_improvement_20260911_r1/conditions/both_yolo26x_Qwen3.8_seed42/best.pt"
36
+ }
37
+ ]
releases/20260913_lr/paper_data/cost_20260911/yolo26x_specialist.json ADDED
@@ -0,0 +1,134 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "component": "specialist",
3
+ "expert": "yolo26x",
4
+ "task": "construction",
5
+ "device": "cuda:0",
6
+ "checkpoint": "/mnt/HithinkOmniSSD/user_workspace/caisihang/research/ICASSP_ModelCollaboration/models/construction/yolo26x.pt",
7
+ "sample_count": 64,
8
+ "batch_size": 16,
9
+ "sample_ids": [
10
+ "0000944",
11
+ "0003511",
12
+ "0003365",
13
+ "0001351",
14
+ "0002149",
15
+ "0003368",
16
+ "0000176",
17
+ "0004487",
18
+ "0000013",
19
+ "0003583",
20
+ "0000758",
21
+ "0001291",
22
+ "0001451",
23
+ "0002755",
24
+ "0003088",
25
+ "0004538",
26
+ "0004324",
27
+ "0000516",
28
+ "0003893",
29
+ "0003475",
30
+ "0002137",
31
+ "0003272",
32
+ "0001711",
33
+ "0003262",
34
+ "0001470",
35
+ "0002145",
36
+ "0002662",
37
+ "0000756",
38
+ "0001631",
39
+ "0001924",
40
+ "0002496",
41
+ "0001443",
42
+ "0004053",
43
+ "0002168",
44
+ "0001881",
45
+ "0001666",
46
+ "0000304",
47
+ "0001252",
48
+ "0004844",
49
+ "0000949",
50
+ "0003430",
51
+ "0003807",
52
+ "0004199",
53
+ "0002538",
54
+ "0002091",
55
+ "0001569",
56
+ "0003048",
57
+ "0002854",
58
+ "0000380",
59
+ "0001234",
60
+ "0001484",
61
+ "0003180",
62
+ "0000824",
63
+ "0003784",
64
+ "0000894",
65
+ "0004168",
66
+ "0000564",
67
+ "0004296",
68
+ "0003054",
69
+ "0001583",
70
+ "0003386",
71
+ "0000205",
72
+ "0004214",
73
+ "0003625"
74
+ ],
75
+ "model_load_seconds_excluded_from_latency": true,
76
+ "forward_latency_ms_per_sample": [
77
+ 28.76109504699707,
78
+ 28.29251480102539,
79
+ 28.306396484375,
80
+ 42.96271896362305
81
+ ],
82
+ "end_to_end_latency_ms_per_sample": [
83
+ 28.19202906539431,
84
+ 27.162021433468908,
85
+ 28.371096690534614,
86
+ 43.443599628517404
87
+ ],
88
+ "mean_forward_latency_ms_per_sample": 32.08068132400513,
89
+ "median_forward_latency_ms_per_sample": 28.533745765686035,
90
+ "mean_end_to_end_latency_ms_per_sample": 31.79218670447881,
91
+ "median_end_to_end_latency_ms_per_sample": 28.28156287796446,
92
+ "flop_profiles": [
93
+ {
94
+ "batch_size": 16,
95
+ "flops_counted": 3110687539200,
96
+ "gflops_counted_per_sample": 194.4179712,
97
+ "operators": {
98
+ "aten.convolution": 3101840179200,
99
+ "aten.bmm": 8847360000
100
+ }
101
+ },
102
+ {
103
+ "batch_size": 16,
104
+ "flops_counted": 3110687539200,
105
+ "gflops_counted_per_sample": 194.4179712,
106
+ "operators": {
107
+ "aten.convolution": 3101840179200,
108
+ "aten.bmm": 8847360000
109
+ }
110
+ },
111
+ {
112
+ "batch_size": 16,
113
+ "flops_counted": 3110687539200,
114
+ "gflops_counted_per_sample": 194.4179712,
115
+ "operators": {
116
+ "aten.convolution": 3101840179200,
117
+ "aten.bmm": 8847360000
118
+ }
119
+ },
120
+ {
121
+ "batch_size": 16,
122
+ "flops_counted": 3110687539200,
123
+ "gflops_counted_per_sample": 194.4179712,
124
+ "operators": {
125
+ "aten.convolution": 3101840179200,
126
+ "aten.bmm": 8847360000
127
+ }
128
+ }
129
+ ],
130
+ "mean_counted_gflops_per_sample": 194.4179712,
131
+ "flops_scope": "GPU FlopCounterMode around the real Ultralytics predict path; multiply-add counted as 2 FLOPs. Elementwise, postprocessing and unsupported custom ops may be missing.",
132
+ "latency_scope": "A800 wall-clock latency for real Ultralytics prediction including image loading/preprocess, forward and result postprocessing, excluding model load; batch processing uses the recorded batch size.",
133
+ "completed": true
134
+ }
releases/20260913_lr/paper_data/cost_20260911/成本性能补充报告.md ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # A800成本—性能补充报告(2026-09-11)
2
+
3
+ ## 1. 做了什么
4
+
5
+ - 在A800上补测了正文中两个代表性模型对:gRefCOCO的InstanceVG + MiniCPM,以及Construction的YOLO26x + Qwen3.8。
6
+ - 分别实测了小模型端、路由器端和大模型端的延迟与可计数运算量;大模型端延迟采用批处理均摊后的真实墙钟时间。
7
+ - 从已经锁定的测试曲线中提取代表性调用预算,按测试集实际调用率换算每个输入的成本,并生成完整曲线。没有用测试集重新挑选模型、阈值或预算。
8
+
9
+ ## 2. 为什么做
10
+
11
+ 正文原有成本图只是代理估算,缺少InstanceVG、完整解码和真实延迟。此次补测用于回答一个更直接的问题:两项同时改相对置信度路由的性能提升,是否值得它增加的计算和时间成本。
12
+
13
+ ## 3. 原始实测成本
14
+
15
+ | 模型对 | 小模型端到端延迟(毫秒/样本) | 小模型可计数运算量(GFLOPs/样本) | 路由器延迟(毫秒/样本) | 路由器稠密运算量(GFLOPs/样本) | 大模型调用延迟(毫秒/样本) | 大模型完整生成估算(GFLOPs/调用) |
16
+ |---|---:|---:|---:|---:|---:|---:|
17
+ | gRefCOCO / InstanceVG + MiniCPM | 16.317 | 7.503 | 0.001831 | 0.000954 | 1447.432 | 10085.742 |
18
+ | Construction / YOLO26x + Qwen3.8 | 31.792 | 194.418 | 0.004676 | 0.000727 | 2599.871 | 94701.391 |
19
+
20
+ 说明:GFLOPs按乘加计2次运算,只覆盖计数器能识别的稠密乘法、卷积和注意力等算子;路由器的计数器返回0,因此按已加载检查点的线性层结构精确计算了稠密乘加量。大模型完整生成GFLOPs为“预填充GFLOPs + 平均生成长度 × 单Token解码GFLOPs”的估算,不把未计数算子冒充成完整硬件计数。
21
+
22
+ ## 4. 验证集锁定点在测试集上的结果
23
+
24
+ 下表的均值和标准差只在三个随机种子之间统计。成本使用对应的平均实际调用率;星号表示验证集选出的运行点。
25
+
26
+ | 模型对 | 方法 | 测试性能 | 实际调用率 | 估算GFLOPs/输入 | 端到端延迟(毫秒/输入) |
27
+ |---|---|---:|---:|---:|---:|
28
+ | gRefCOCO / InstanceVG + MiniCPM | 两项同时改* | 68.05±0.04% | 11.29±1.28% | 1146.415 | 179.767 |
29
+ | gRefCOCO / InstanceVG + MiniCPM | 原方法* | 67.33±0.15% | 26.05±1.96% | 2634.360 | 393.306 |
30
+ | gRefCOCO / InstanceVG + MiniCPM | 置信度路由* | 61.88±0.00% | 52.09±0.00% | 5261.597 | 770.348 |
31
+ | Construction / YOLO26x + Qwen3.8 | 两项同时改* | 30.11±0.14% | 44.14±0.58% | 41996.697 | 1179.410 |
32
+ | Construction / YOLO26x + Qwen3.8 | 原方法* | 30.00±0.22% | 46.67±1.72% | 44392.604 | 1245.185 |
33
+ | Construction / YOLO26x + Qwen3.8 | 置信度路由* | 25.97±0.00% | 92.21±0.00% | 87518.937 | 2429.148 |
34
+
35
+ ### 两项同时改相对置信度路由
36
+
37
+ | 模型对 | 性能变化 | 调用率变化 | 估算GFLOPs变化 | 实际延迟变化 |
38
+ |---|---:|---:|---:|---:|
39
+ | gRefCOCO / InstanceVG + MiniCPM | +6.18 个百分点 | -40.80 个百分点 | -78.2% | -76.7% |
40
+ | Construction / YOLO26x + Qwen3.8 | +4.13 个百分点 | -48.07 个百分点 | -52.0% | -51.4% |
41
+
42
+ 直白地说:两项同时改在这两个代表性模型对上都提高了测试性能,同时减少了实际调用,因此计算量和延迟也一起下降。原因不是路由器本身更快,而是它让大模型少被调用;路由器自身成本相对很小。
43
+
44
+ ## 5. 固定名义调用预算的代表点
45
+
46
+ 名义预算是验证阶段设定的阈值预算;阈值原样转到测试集后,实际调用率会有偏差。完整的99个阈值点和端点均在CSV中。
47
+
48
+ | 模型对 | 名义预算 | 两项同时改:性能 / 实际调用 | 置信度:性能 / 实际调用 | 两项同时改:GFLOPs/输入 | 两项同时改:毫秒/输入 |
49
+ |---|---:|---:|---:|---:|---:|
50
+ | gRefCOCO / InstanceVG + MiniCPM | 25% | 67.94% / 24.53% | 65.97% / 12.48% | 2481.628 | 371.387 |
51
+ | gRefCOCO / InstanceVG + MiniCPM | 50% | 66.46% / 43.10% | 65.28% / 23.45% | 4354.834 | 640.216 |
52
+ | Construction / YOLO26x + Qwen3.8 | 25% | 25.62% / 26.50% | 18.03% / 24.80% | 25288.396 | 720.711 |
53
+ | Construction / YOLO26x + Qwen3.8 | 50% | 29.98% / 49.51% | 18.02% / 50.13% | 47082.746 | 1319.039 |
54
+
55
+ ## 6. 结果怎么理解
56
+
57
+ 1. gRefCOCO的InstanceVG + MiniCPM是这两组中相对置信度提升更明显的一组;Construction的YOLO26x + Qwen3.8也有稳定的正向变化。
58
+ 2. 这里的成本是按真实测试调用率换算的平均每输入成本,不是把名义预算当成实际调用率。
59
+ 3. 大模型延迟是A800、批量4下的均摊值,模型加载和预热没有计入;实际系统的并发度、批量大小和请求排队会改变墙钟延迟。
60
+ 4. InstanceVG使用了远端已有的MMCV等价算子兼容路径,因为原生detrex扩展不可用;这不改变本次模型输出链路,但应在复现实验时保留该环境说明。
61
+ 5. 本补充不修改正文,也不覆盖旧实验结果;它是针对两个代表性模型对的成本实测和成本—性能换算。
62
+
63
+ ## 7. 文件在哪里
64
+
65
+ - 原始A800测量记录:`C:/Users/csh10/Documents/CodexProjects/ICASSP-COLLA/dse_results/method_improvement_20260911/cost_analysis_20260911`目录下的8个JSON文件。
66
+ - 完整成本—性能曲线数据:`C:/Users/csh10/Documents/CodexProjects/ICASSP-COLLA/dse_results/method_improvement_20260911/cost_analysis_20260911/full_cost_performance_curves.csv`。
67
+ - 固定预算代表点:`C:/Users/csh10/Documents/CodexProjects/ICASSP-COLLA/dse_results/method_improvement_20260911/cost_analysis_20260911/fixed_budget_cost_performance.csv`。
68
+ - FLOPs—性能图:`C:/Users/csh10/Documents/CodexProjects/ICASSP-COLLA/dse_results/method_improvement_20260911/cost_analysis_20260911/flops_performance.pdf`;实际延迟—性能图:`C:/Users/csh10/Documents/CodexProjects/ICASSP-COLLA/dse_results/method_improvement_20260911/cost_analysis_20260911/latency_performance.pdf`。
69
+ - 可复现脚本:`C:/Users/csh10/Documents/CodexProjects/ICASSP-COLLA/dse_results/method_improvement_20260911/cost_analysis_20260911/make_cost_analysis_20260911.py`;插入论文的LaTeX片段:`C:/Users/csh10/Documents/CodexProjects/ICASSP-COLLA/dse_results/method_improvement_20260911/cost_analysis_20260911/latex_includes.tex`。
releases/20260913_lr/paper_data/cost_components.json ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "components": {
3
+ "grefcoco|instancevg|MiniCPM": {
4
+ "specialist_gflops": 7.502680064,
5
+ "router_gflops": 3.642171392,
6
+ "large_gflops": 12916.147709832,
7
+ "specialist_ms": 46.08042127620138,
8
+ "router_ms": 13.180577338061994,
9
+ "large_ms": 3667.173011115665,
10
+ "large_samples": 64,
11
+ "flops_probes": 8,
12
+ "specialist_samples": 64,
13
+ "router_samples": 64,
14
+ "batch_size": 1
15
+ },
16
+ "construction|yolo26x|Qwen3.8": {
17
+ "specialist_gflops": 135.97853184000002,
18
+ "router_gflops": 3.183754752,
19
+ "large_gflops": 96874.712551641,
20
+ "specialist_ms": 46.30809425361804,
21
+ "router_ms": 25.261267359383055,
22
+ "large_ms": 9822.475872695577,
23
+ "large_samples": 64,
24
+ "flops_probes": 8,
25
+ "specialist_samples": 64,
26
+ "router_samples": 64,
27
+ "batch_size": 1
28
+ }
29
+ },
30
+ "protocol": {
31
+ "method": "original main-table logit contrast; original checkpoints and validation thresholds; seed 42",
32
+ "large_timing": "64 measured examples; batch size one; same full task prompt; two separate warmup examples",
33
+ "cost": "serial component estimates, not a timed integrated service; includes active distilled encoders and routing head only for learned",
34
+ "flops": "multiply-add=2; supported operators; mean prefill (including visual embedding) plus average generated length times one-token decode probe",
35
+ "scope_limits": "mean call cost extrapolated across policies; no policy-specific input/output-length costs; specialist-feature extraction overhead is not separately measured; confidence threshold overhead not separately timed"
36
+ },
37
+ "source_files": [
38
+ "dse_results/main_table_cost_20260912/cost_summary.json",
39
+ "dse_results/main_table_cost_qwen_20260912/cost_summary.json"
40
+ ]
41
+ }
releases/20260913_lr/paper_data/cost_five_seeds.json ADDED
@@ -0,0 +1,3299 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "components": {
3
+ "grefcoco|instancevg|MiniCPM": {
4
+ "specialist_gflops": 7.502680064,
5
+ "router_gflops": 3.642171392,
6
+ "large_gflops": 12916.147709832,
7
+ "specialist_ms": 46.08042127620138,
8
+ "router_ms": 13.180577338061994,
9
+ "large_ms": 3667.173011115665,
10
+ "large_samples": 64,
11
+ "flops_probes": 8,
12
+ "specialist_samples": 64,
13
+ "router_samples": 64,
14
+ "batch_size": 1
15
+ },
16
+ "construction|yolo26x|Qwen3.8": {
17
+ "specialist_gflops": 135.97853184000002,
18
+ "router_gflops": 3.183754752,
19
+ "large_gflops": 96874.712551641,
20
+ "specialist_ms": 46.30809425361804,
21
+ "router_ms": 25.261267359383055,
22
+ "large_ms": 9822.475872695577,
23
+ "large_samples": 64,
24
+ "flops_probes": 8,
25
+ "specialist_samples": 64,
26
+ "router_samples": 64,
27
+ "batch_size": 1
28
+ }
29
+ },
30
+ "selected": [
31
+ {
32
+ "pair": "grefcoco|instancevg|MiniCPM",
33
+ "method": "learned",
34
+ "metric": 0.6791367722542041,
35
+ "calls": 0.19038085245157815,
36
+ "gflops": 2470.132062844315,
37
+ "ms": 757.4205225578843
38
+ },
39
+ {
40
+ "pair": "grefcoco|instancevg|MiniCPM",
41
+ "method": "confidence",
42
+ "metric": 0.6187788900547316,
43
+ "calls": 0.5209426310864078,
44
+ "gflops": 6736.074651524562,
45
+ "ms": 1956.4671783358606
46
+ },
47
+ {
48
+ "pair": "construction|yolo26x|Qwen3.8",
49
+ "method": "learned",
50
+ "metric": 0.2979678330127048,
51
+ "calls": 0.44766977363515315,
52
+ "gflops": 43507.04292555566,
53
+ "ms": 4468.794912079384
54
+ },
55
+ {
56
+ "pair": "construction|yolo26x|Qwen3.8",
57
+ "method": "confidence",
58
+ "metric": 0.2597490004762105,
59
+ "calls": 0.9221038615179761,
60
+ "gflops": 89464.5250591521,
61
+ "ms": 9103.651026133362
62
+ }
63
+ ],
64
+ "curves": [
65
+ {
66
+ "pair": "grefcoco|instancevg|MiniCPM",
67
+ "method": "learned",
68
+ "metric": 0.6695119530385957,
69
+ "calls": 0.0,
70
+ "gflops": 11.144851456,
71
+ "ms": 59.26099861426337
72
+ },
73
+ {
74
+ "pair": "grefcoco|instancevg|MiniCPM",
75
+ "method": "learned",
76
+ "metric": 0.6736239117488585,
77
+ "calls": 0.020332926863851627,
78
+ "gflops": 273.76793820271877,
79
+ "ms": 133.82535924636875
80
+ },
81
+ {
82
+ "pair": "grefcoco|instancevg|MiniCPM",
83
+ "method": "learned",
84
+ "metric": 0.6766298953577403,
85
+ "calls": 0.04462467742392876,
86
+ "gflops": 587.5237765670691,
87
+ "ms": 222.90741129303743
88
+ },
89
+ {
90
+ "pair": "grefcoco|instancevg|MiniCPM",
91
+ "method": "learned",
92
+ "metric": 0.6786603522105323,
93
+ "calls": 0.07091285483367836,
94
+ "gflops": 927.0657590136638,
95
+ "ms": 319.31070600149167
96
+ },
97
+ {
98
+ "pair": "grefcoco|instancevg|MiniCPM",
99
+ "method": "learned",
100
+ "metric": 0.6797663273119132,
101
+ "calls": 0.09045174829141027,
102
+ "gflops": 1179.4329930003992,
103
+ "ms": 390.9632087567506
104
+ },
105
+ {
106
+ "pair": "grefcoco|instancevg|MiniCPM",
107
+ "method": "learned",
108
+ "metric": 0.6804072257039957,
109
+ "calls": 0.11004735842100785,
110
+ "gflops": 1432.5327878985618,
111
+ "ms": 462.8237013603555
112
+ },
113
+ {
114
+ "pair": "grefcoco|instancevg|MiniCPM",
115
+ "method": "learned",
116
+ "metric": 0.6805717040524062,
117
+ "calls": 0.12548563650285,
118
+ "gflops": 1631.9358679890966,
119
+ "ms": 519.4385380801856
120
+ },
121
+ {
122
+ "pair": "grefcoco|instancevg|MiniCPM",
123
+ "method": "learned",
124
+ "metric": 0.6804015540368091,
125
+ "calls": 0.13746986926807134,
126
+ "gflops": 1786.725988573704,
127
+ "ms": 563.3867930357333
128
+ },
129
+ {
130
+ "pair": "grefcoco|instancevg|MiniCPM",
131
+ "method": "learned",
132
+ "metric": 0.6803391656977569,
133
+ "calls": 0.15023112043785272,
134
+ "gflops": 1951.5521936448667,
135
+ "ms": 610.1845089136239
136
+ },
137
+ {
138
+ "pair": "grefcoco|instancevg|MiniCPM",
139
+ "method": "learned",
140
+ "metric": 0.6802767773587046,
141
+ "calls": 0.16403595836996285,
142
+ "gflops": 2129.857519486293,
143
+ "ms": 660.8092380010838
144
+ },
145
+ {
146
+ "pair": "grefcoco|instancevg|MiniCPM",
147
+ "method": "learned",
148
+ "metric": 0.6802881206930778,
149
+ "calls": 0.1779372146442447,
150
+ "gflops": 2309.4081988771463,
151
+ "ms": 711.7875498307326
152
+ },
153
+ {
154
+ "pair": "grefcoco|instancevg|MiniCPM",
155
+ "method": "learned",
156
+ "metric": 0.6802824490258912,
157
+ "calls": 0.18960383404701814,
158
+ "gflops": 2460.09597835776,
159
+ "ms": 754.5710616355417
160
+ },
161
+ {
162
+ "pair": "grefcoco|instancevg|MiniCPM",
163
+ "method": "learned",
164
+ "metric": 0.6801293140118538,
165
+ "calls": 0.2002041800187165,
166
+ "gflops": 2597.011612703539,
167
+ "ms": 793.4443642914426
168
+ },
169
+ {
170
+ "pair": "grefcoco|instancevg|MiniCPM",
171
+ "method": "learned",
172
+ "metric": 0.6800839406743613,
173
+ "calls": 0.2103564642826759,
174
+ "gflops": 2728.1400158490414,
175
+ "ms": 830.6745471454087
176
+ },
177
+ {
178
+ "pair": "grefcoco|instancevg|MiniCPM",
179
+ "method": "learned",
180
+ "metric": 0.6799818506650028,
181
+ "calls": 0.21922695176247053,
182
+ "gflops": 2842.7125423962843,
183
+ "ms": 863.2041594267511
184
+ },
185
+ {
186
+ "pair": "grefcoco|instancevg|MiniCPM",
187
+ "method": "learned",
188
+ "metric": 0.6798684173212717,
189
+ "calls": 0.2273317641720784,
190
+ "gflops": 2947.395496639259,
191
+ "ms": 892.9259087554203
192
+ },
193
+ {
194
+ "pair": "grefcoco|instancevg|MiniCPM",
195
+ "method": "learned",
196
+ "metric": 0.6796925956384878,
197
+ "calls": 0.2348580665286561,
198
+ "gflops": 3044.606329585673,
199
+ "ms": 920.5261616309583
200
+ },
201
+ {
202
+ "pair": "grefcoco|instancevg|MiniCPM",
203
+ "method": "learned",
204
+ "metric": 0.6796018489635027,
205
+ "calls": 0.24202705385247994,
206
+ "gflops": 3137.202028790095,
207
+ "ms": 946.8160784619154
208
+ },
209
+ {
210
+ "pair": "grefcoco|instancevg|MiniCPM",
211
+ "method": "learned",
212
+ "metric": 0.6794316989479057,
213
+ "calls": 0.24939454952783371,
214
+ "gflops": 3232.361691184513,
215
+ "ms": 973.8339597620842
216
+ },
217
+ {
218
+ "pair": "grefcoco|instancevg|MiniCPM",
219
+ "method": "learned",
220
+ "metric": 0.6793693106088534,
221
+ "calls": 0.25641040183762015,
222
+ "gflops": 3322.9794759280803,
223
+ "ms": 999.5623040025065
224
+ },
225
+ {
226
+ "pair": "grefcoco|instancevg|MiniCPM",
227
+ "method": "learned",
228
+ "metric": 0.6792615489323086,
229
+ "calls": 0.2633184924708618,
230
+ "gflops": 3412.2053949399365,
231
+ "ms": 1024.895467531071
232
+ },
233
+ {
234
+ "pair": "grefcoco|instancevg|MiniCPM",
235
+ "method": "learned",
236
+ "metric": 0.6791027422510847,
237
+ "calls": 0.2692510563480135,
238
+ "gflops": 3488.8312662752414,
239
+ "ms": 1046.6512056680817
240
+ },
241
+ {
242
+ "pair": "grefcoco|instancevg|MiniCPM",
243
+ "method": "learned",
244
+ "metric": 0.6789496072370473,
245
+ "calls": 0.27534242690638916,
246
+ "gflops": 3567.508308162543,
247
+ "ms": 1068.9893153804614
248
+ },
249
+ {
250
+ "pair": "grefcoco|instancevg|MiniCPM",
251
+ "method": "learned",
252
+ "metric": 0.6787681138870771,
253
+ "calls": 0.2815245441397499,
254
+ "gflops": 3647.3574475081286,
255
+ "ms": 1091.660208850195
256
+ },
257
+ {
258
+ "pair": "grefcoco|instancevg|MiniCPM",
259
+ "method": "learned",
260
+ "metric": 0.6786490088761592,
261
+ "calls": 0.28715650965601336,
262
+ "gflops": 3720.1007460128676,
263
+ "ms": 1112.3136007909704
264
+ },
265
+ {
266
+ "pair": "grefcoco|instancevg|MiniCPM",
267
+ "method": "learned",
268
+ "metric": 0.6784902021949353,
269
+ "calls": 0.29289056518163514,
270
+ "gflops": 3794.162654158177,
271
+ "ms": 1133.3413744587692
272
+ },
273
+ {
274
+ "pair": "grefcoco|instancevg|MiniCPM",
275
+ "method": "learned",
276
+ "metric": 0.6783540821824576,
277
+ "calls": 0.2986983523806823,
278
+ "gflops": 3869.1768914883414,
279
+ "ms": 1154.639534929418
280
+ },
281
+ {
282
+ "pair": "grefcoco|instancevg|MiniCPM",
283
+ "method": "learned",
284
+ "metric": 0.6781215438278082,
285
+ "calls": 0.30438703456881144,
286
+ "gflops": 3942.6527509045077,
287
+ "ms": 1175.5009167185397
288
+ },
289
+ {
290
+ "pair": "grefcoco|instancevg|MiniCPM",
291
+ "method": "learned",
292
+ "metric": 0.6777301987919349,
293
+ "calls": 0.31002467175226156,
294
+ "gflops": 4015.469305500391,
295
+ "ms": 1196.17510764415
296
+ },
297
+ {
298
+ "pair": "grefcoco|instancevg|MiniCPM",
299
+ "method": "learned",
300
+ "metric": 0.6775146754388452,
301
+ "calls": 0.31561126393103256,
302
+ "gflops": 4087.626555275989,
303
+ "ms": 1216.662107706249
304
+ },
305
+ {
306
+ "pair": "grefcoco|instancevg|MiniCPM",
307
+ "method": "learned",
308
+ "metric": 0.6773048237529421,
309
+ "calls": 0.3206477043927063,
310
+ "gflops": 4152.677964210741,
311
+ "ms": 1235.1316062393896
312
+ },
313
+ {
314
+ "pair": "grefcoco|instancevg|MiniCPM",
315
+ "method": "learned",
316
+ "metric": 0.6772084054107704,
317
+ "calls": 0.3267560899526416,
318
+ "gflops": 4231.57477437147,
319
+ "ms": 1257.5321129062731
320
+ },
321
+ {
322
+ "pair": "grefcoco|instancevg|MiniCPM",
323
+ "method": "learned",
324
+ "metric": 0.6769191503842554,
325
+ "calls": 0.3326829821626067,
326
+ "gflops": 4308.127389615633,
327
+ "ms": 1279.2670520584488
328
+ },
329
+ {
330
+ "pair": "grefcoco|instancevg|MiniCPM",
331
+ "method": "learned",
332
+ "metric": 0.6765788503530613,
333
+ "calls": 0.33824688767263134,
334
+ "gflops": 4379.991615026658,
335
+ "ms": 1299.6708561812088
336
+ },
337
+ {
338
+ "pair": "grefcoco|instancevg|MiniCPM",
339
+ "method": "learned",
340
+ "metric": 0.6762782519921732,
341
+ "calls": 0.3451209483027536,
342
+ "gflops": 4468.777997491658,
343
+ "ms": 1324.8792258007659
344
+ },
345
+ {
346
+ "pair": "grefcoco|instancevg|MiniCPM",
347
+ "method": "learned",
348
+ "metric": 0.6760173553015909,
349
+ "calls": 0.3511272438533307,
350
+ "gflops": 4546.356198011819,
351
+ "ms": 1346.9053507406265
352
+ },
353
+ {
354
+ "pair": "grefcoco|instancevg|MiniCPM",
355
+ "method": "learned",
356
+ "metric": 0.6754955619204265,
357
+ "calls": 0.35775175112724383,
358
+ "gflops": 4631.919312466537,
359
+ "ms": 1371.19856502746
360
+ },
361
+ {
362
+ "pair": "grefcoco|instancevg|MiniCPM",
363
+ "method": "learned",
364
+ "metric": 0.6750021268751949,
365
+ "calls": 0.36507387346510506,
366
+ "gflops": 4726.492926131814,
367
+ "ms": 1398.0500544489519
368
+ },
369
+ {
370
+ "pair": "grefcoco|instancevg|MiniCPM",
371
+ "method": "learned",
372
+ "metric": 0.6744406318237245,
373
+ "calls": 0.3725207724810708,
374
+ "gflops": 4822.678173802229,
375
+ "ms": 1425.3591215368053
376
+ },
377
+ {
378
+ "pair": "grefcoco|instancevg|MiniCPM",
379
+ "method": "learned",
380
+ "metric": 0.6740152567847318,
381
+ "calls": 0.37905453307999887,
382
+ "gflops": 4907.069190798665,
383
+ "ms": 1449.3195520662853
384
+ },
385
+ {
386
+ "pair": "grefcoco|instancevg|MiniCPM",
387
+ "method": "learned",
388
+ "metric": 0.6733346567223435,
389
+ "calls": 0.38662053710688254,
390
+ "gflops": 5004.792816383078,
391
+ "ms": 1477.0653978356654
392
+ },
393
+ {
394
+ "pair": "grefcoco|instancevg|MiniCPM",
395
+ "method": "learned",
396
+ "metric": 0.6728752516802314,
397
+ "calls": 0.39437937781810967,
398
+ "gflops": 5105.007149066346,
399
+ "ms": 1505.5184090894231
400
+ },
401
+ {
402
+ "pair": "grefcoco|instancevg|MiniCPM",
403
+ "method": "learned",
404
+ "metric": 0.6721946516178432,
405
+ "calls": 0.40177523182939623,
406
+ "gflops": 5200.533091916476,
407
+ "ms": 1532.6402853137647
408
+ },
409
+ {
410
+ "pair": "grefcoco|instancevg|MiniCPM",
411
+ "method": "learned",
412
+ "metric": 0.6716388282335591,
413
+ "calls": 0.4089045174829141,
414
+ "gflops": 5292.6159984829,
415
+ "ms": 1558.7846092508796
416
+ },
417
+ {
418
+ "pair": "grefcoco|instancevg|MiniCPM",
419
+ "method": "learned",
420
+ "metric": 0.6709298698352381,
421
+ "calls": 0.4166690298613277,
422
+ "gflops": 5392.903587257309,
423
+ "ms": 1587.2584194894714
424
+ },
425
+ {
426
+ "pair": "grefcoco|instancevg|MiniCPM",
427
+ "method": "learned",
428
+ "metric": 0.670351359782208,
429
+ "calls": 0.42493832061934605,
430
+ "gflops": 5499.710968143422,
431
+ "ms": 1617.5833393783444
432
+ },
433
+ {
434
+ "pair": "grefcoco|instancevg|MiniCPM",
435
+ "method": "learned",
436
+ "metric": 0.6695743413776479,
437
+ "calls": 0.43286731134617024,
438
+ "gflops": 5602.122983560971,
439
+ "ms": 1646.6603201771404
440
+ },
441
+ {
442
+ "pair": "grefcoco|instancevg|MiniCPM",
443
+ "method": "learned",
444
+ "metric": 0.6687065762981028,
445
+ "calls": 0.4409891387573377,
446
+ "gflops": 5707.025706077373,
447
+ "ms": 1676.4444664603132
448
+ },
449
+ {
450
+ "pair": "grefcoco|instancevg|MiniCPM",
451
+ "method": "learned",
452
+ "metric": 0.6676913478717068,
453
+ "calls": 0.4495646995434308,
454
+ "gflops": 5817.788915885195,
455
+ "ms": 1707.8925315302556
456
+ },
457
+ {
458
+ "pair": "grefcoco|instancevg|MiniCPM",
459
+ "method": "learned",
460
+ "metric": 0.6667838811218558,
461
+ "calls": 0.45795309531236705,
462
+ "gflops": 5926.134674685305,
463
+ "ms": 1738.6542301006555
464
+ },
465
+ {
466
+ "pair": "grefcoco|instancevg|MiniCPM",
467
+ "method": "learned",
468
+ "metric": 0.665910444375124,
469
+ "calls": 0.46621671440319884,
470
+ "gflops": 6032.868799480276,
471
+ "ms": 1768.9583510046941
472
+ },
473
+ {
474
+ "pair": "grefcoco|instancevg|MiniCPM",
475
+ "method": "learned",
476
+ "metric": 0.6649065592831013,
477
+ "calls": 0.47530839690326976,
478
+ "gflops": 6150.298313582086,
479
+ "ms": 1802.2991236945866
480
+ },
481
+ {
482
+ "pair": "grefcoco|instancevg|MiniCPM",
483
+ "method": "learned",
484
+ "metric": 0.6638005841817202,
485
+ "calls": 0.4846496327595496,
486
+ "gflops": 6270.951095694176,
487
+ "ms": 1836.555051717202
488
+ },
489
+ {
490
+ "pair": "grefcoco|instancevg|MiniCPM",
491
+ "method": "learned",
492
+ "metric": 0.6626946090803392,
493
+ "calls": 0.4937072852565011,
494
+ "gflops": 6387.94107324913,
495
+ "ms": 1869.771030498087
496
+ },
497
+ {
498
+ "pair": "grefcoco|instancevg|MiniCPM",
499
+ "method": "learned",
500
+ "metric": 0.6615659473102118,
501
+ "calls": 0.5040467345376174,
502
+ "gflops": 6521.486927402344,
503
+ "ms": 1907.687579851596
504
+ },
505
+ {
506
+ "pair": "grefcoco|instancevg|MiniCPM",
507
+ "method": "learned",
508
+ "metric": 0.6605280322150696,
509
+ "calls": 0.5130249836939569,
510
+ "gflops": 6637.4513196812995,
511
+ "ms": 1940.612372844796
512
+ },
513
+ {
514
+ "pair": "grefcoco|instancevg|MiniCPM",
515
+ "method": "learned",
516
+ "metric": 0.6591214587528004,
517
+ "calls": 0.5229674162720132,
518
+ "gflops": 6765.8692474545205,
519
+ "ms": 1977.0729932598813
520
+ },
521
+ {
522
+ "pair": "grefcoco|instancevg|MiniCPM",
523
+ "method": "learned",
524
+ "metric": 0.6579020503076879,
525
+ "calls": 0.5324504438079574,
526
+ "gflops": 6888.35343184518,
527
+ "ms": 2011.8488959033625
528
+ },
529
+ {
530
+ "pair": "grefcoco|instancevg|MiniCPM",
531
+ "method": "learned",
532
+ "metric": 0.6562969684938889,
533
+ "calls": 0.5418654113376628,
534
+ "gflops": 7009.9585431421265,
535
+ "ms": 2046.3752107288285
536
+ },
537
+ {
538
+ "pair": "grefcoco|instancevg|MiniCPM",
539
+ "method": "learned",
540
+ "metric": 0.6548733800300598,
541
+ "calls": 0.5520120239344355,
542
+ "gflops": 7141.013690196485,
543
+ "ms": 2083.58459459796
544
+ },
545
+ {
546
+ "pair": "grefcoco|instancevg|MiniCPM",
547
+ "method": "learned",
548
+ "metric": 0.6530811331991039,
549
+ "calls": 0.5628449082607834,
550
+ "gflops": 7280.932824279119,
551
+ "ms": 2123.3106556320804
552
+ },
553
+ {
554
+ "pair": "grefcoco|instancevg|MiniCPM",
555
+ "method": "learned",
556
+ "metric": 0.6512435130306553,
557
+ "calls": 0.5729631625216232,
558
+ "gflops": 7411.621690877762,
559
+ "ms": 2160.416044577038
560
+ },
561
+ {
562
+ "pair": "grefcoco|instancevg|MiniCPM",
563
+ "method": "learned",
564
+ "metric": 0.6493945495278337,
565
+ "calls": 0.5823667867169554,
566
+ "gflops": 7533.080289992424,
567
+ "ms": 2194.9007614328348
568
+ },
569
+ {
570
+ "pair": "grefcoco|instancevg|MiniCPM",
571
+ "method": "learned",
572
+ "metric": 0.647755437710915,
573
+ "calls": 0.5923942943028104,
574
+ "gflops": 7662.597059132788,
575
+ "ms": 2231.67336662044
576
+ },
577
+ {
578
+ "pair": "grefcoco|instancevg|MiniCPM",
579
+ "method": "learned",
580
+ "metric": 0.6456966225221904,
581
+ "calls": 0.6029662819385759,
582
+ "gflops": 7799.146413022853,
583
+ "ms": 2270.4426743521676
584
+ },
585
+ {
586
+ "pair": "grefcoco|instancevg|MiniCPM",
587
+ "method": "learned",
588
+ "metric": 0.6438760173553015,
589
+ "calls": 0.6129314011853785,
590
+ "gflops": 7927.857365160644,
591
+ "ms": 2306.986490706591
592
+ },
593
+ {
594
+ "pair": "grefcoco|instancevg|MiniCPM",
595
+ "method": "learned",
596
+ "metric": 0.641459887133823,
597
+ "calls": 0.6240138388679352,
598
+ "gflops": 8070.999767253555,
599
+ "ms": 2347.6277070734345
600
+ },
601
+ {
602
+ "pair": "grefcoco|instancevg|MiniCPM",
603
+ "method": "learned",
604
+ "metric": 0.6394010719450983,
605
+ "calls": 0.6343873181521709,
606
+ "gflops": 8204.985157953626,
607
+ "ms": 2385.669050335951
608
+ },
609
+ {
610
+ "pair": "grefcoco|instancevg|MiniCPM",
611
+ "method": "learned",
612
+ "metric": 0.6372458384142019,
613
+ "calls": 0.6447324391004736,
614
+ "gflops": 8338.604268197982,
615
+ "ms": 2423.6063986742943
616
+ },
617
+ {
618
+ "pair": "grefcoco|instancevg|MiniCPM",
619
+ "method": "learned",
620
+ "metric": 0.6348240365255367,
621
+ "calls": 0.655871593454896,
622
+ "gflops": 8482.479231202318,
623
+ "ms": 2464.4556048894833
624
+ },
625
+ {
626
+ "pair": "grefcoco|instancevg|MiniCPM",
627
+ "method": "learned",
628
+ "metric": 0.6324419363071774,
629
+ "calls": 0.6665116410969005,
630
+ "gflops": 8619.907658186099,
631
+ "ms": 2503.4745004392275
632
+ },
633
+ {
634
+ "pair": "grefcoco|instancevg|MiniCPM",
635
+ "method": "learned",
636
+ "metric": 0.6297138643904374,
637
+ "calls": 0.6780421404871962,
638
+ "gflops": 8768.837291479285,
639
+ "ms": 2545.7588366080054
640
+ },
641
+ {
642
+ "pair": "grefcoco|instancevg|MiniCPM",
643
+ "method": "learned",
644
+ "metric": 0.6270595241471231,
645
+ "calls": 0.687848453052775,
646
+ "gflops": 8895.497073065082,
647
+ "ms": 2581.7202813870604
648
+ },
649
+ {
650
+ "pair": "grefcoco|instancevg|MiniCPM",
651
+ "method": "learned",
652
+ "metric": 0.624359810566316,
653
+ "calls": 0.6985792473697643,
654
+ "gflops": 9034.097597507143,
655
+ "ms": 2621.0719606941566
656
+ },
657
+ {
658
+ "pair": "grefcoco|instancevg|MiniCPM",
659
+ "method": "learned",
660
+ "metric": 0.62159203697927,
661
+ "calls": 0.7093157133539403,
662
+ "gflops": 9172.771378040346,
663
+ "ms": 2660.4444389860882
664
+ },
665
+ {
666
+ "pair": "grefcoco|instancevg|MiniCPM",
667
+ "method": "learned",
668
+ "metric": 0.6185803817032016,
669
+ "calls": 0.7205569577177211,
670
+ "gflops": 9317.964950685257,
671
+ "ms": 2701.6680269283015
672
+ },
673
+ {
674
+ "pair": "grefcoco|instancevg|MiniCPM",
675
+ "method": "learned",
676
+ "metric": 0.6154042480787226,
677
+ "calls": 0.7306752119785611,
678
+ "gflops": 9448.653817283903,
679
+ "ms": 2738.77341587326
680
+ },
681
+ {
682
+ "pair": "grefcoco|instancevg|MiniCPM",
683
+ "method": "learned",
684
+ "metric": 0.6121713977823781,
685
+ "calls": 0.7409692879221847,
686
+ "gflops": 9581.613622707973,
687
+ "ms": 2776.5235733480918
688
+ },
689
+ {
690
+ "pair": "grefcoco|instancevg|MiniCPM",
691
+ "method": "learned",
692
+ "metric": 0.6087967558063693,
693
+ "calls": 0.7520970989422342,
694
+ "gflops": 9725.342073530028,
695
+ "ms": 2817.3311815936127
696
+ },
697
+ {
698
+ "pair": "grefcoco|instancevg|MiniCPM",
699
+ "method": "learned",
700
+ "metric": 0.6053710688256813,
701
+ "calls": 0.7639338683606047,
702
+ "gflops": 9878.227535744925,
703
+ "ms": 2860.73866294346
704
+ },
705
+ {
706
+ "pair": "grefcoco|instancevg|MiniCPM",
707
+ "method": "learned",
708
+ "metric": 0.6012931401185378,
709
+ "calls": 0.7769787028897144,
710
+ "gflops": 10046.716545373221,
711
+ "ms": 2908.576328063081
712
+ },
713
+ {
714
+ "pair": "grefcoco|instancevg|MiniCPM",
715
+ "method": "learned",
716
+ "metric": 0.5976405864503871,
717
+ "calls": 0.7888154723080849,
718
+ "gflops": 10199.602007588117,
719
+ "ms": 2951.9838094129286
720
+ },
721
+ {
722
+ "pair": "grefcoco|instancevg|MiniCPM",
723
+ "method": "learned",
724
+ "metric": 0.5939540027791168,
725
+ "calls": 0.799671043303179,
726
+ "gflops": 10339.81416603532,
727
+ "ms": 2991.793066386387
728
+ },
729
+ {
730
+ "pair": "grefcoco|instancevg|MiniCPM",
731
+ "method": "learned",
732
+ "metric": 0.5896775657204435,
733
+ "calls": 0.8114284093809374,
734
+ "gflops": 10491.674042974217,
735
+ "ms": 3034.90936194855
736
+ },
737
+ {
738
+ "pair": "grefcoco|instancevg|MiniCPM",
739
+ "method": "learned",
740
+ "metric": 0.5853273969883448,
741
+ "calls": 0.823225477129002,
742
+ "gflops": 10644.046712551113,
743
+ "ms": 3078.1712504045554
744
+ },
745
+ {
746
+ "pair": "grefcoco|instancevg|MiniCPM",
747
+ "method": "learned",
748
+ "metric": 0.5808467799109548,
749
+ "calls": 0.8346992598474321,
750
+ "gflops": 10792.243784932874,
751
+ "ms": 3120.2475967249875
752
+ },
753
+ {
754
+ "pair": "grefcoco|instancevg|MiniCPM",
755
+ "method": "learned",
756
+ "metric": 0.5766554178600799,
757
+ "calls": 0.8458951308737204,
758
+ "gflops": 10936.851308848642,
759
+ "ms": 3161.304792788524
760
+ },
761
+ {
762
+ "pair": "grefcoco|instancevg|MiniCPM",
763
+ "method": "learned",
764
+ "metric": 0.5724016674701529,
765
+ "calls": 0.8569038368828519,
766
+ "gflops": 11079.0413817567,
767
+ "ms": 3201.6756223525176
768
+ },
769
+ {
770
+ "pair": "grefcoco|instancevg|MiniCPM",
771
+ "method": "learned",
772
+ "metric": 0.5677111987068598,
773
+ "calls": 0.868638516291864,
774
+ "gflops": 11230.608234331025,
775
+ "ms": 3244.7087219753416
776
+ },
777
+ {
778
+ "pair": "grefcoco|instancevg|MiniCPM",
779
+ "method": "learned",
780
+ "metric": 0.5631284916201118,
781
+ "calls": 0.8793125939369878,
782
+ "gflops": 11368.476197861659,
783
+ "ms": 3283.852411434093
784
+ },
785
+ {
786
+ "pair": "grefcoco|instancevg|MiniCPM",
787
+ "method": "learned",
788
+ "metric": 0.5585287695318039,
789
+ "calls": 0.8898675665711936,
790
+ "gflops": 11504.805783478296,
791
+ "ms": 3322.5593222113166
792
+ },
793
+ {
794
+ "pair": "grefcoco|instancevg|MiniCPM",
795
+ "method": "learned",
796
+ "metric": 0.5539857641153618,
797
+ "calls": 0.9006267192241161,
798
+ "gflops": 11643.772588376074,
799
+ "ms": 3362.0149964425877
800
+ },
801
+ {
802
+ "pair": "grefcoco|instancevg|MiniCPM",
803
+ "method": "learned",
804
+ "metric": 0.5498227604004197,
805
+ "calls": 0.9102855684428437,
806
+ "gflops": 11768.527711592154,
807
+ "ms": 3397.435667615941
808
+ },
809
+ {
810
+ "pair": "grefcoco|instancevg|MiniCPM",
811
+ "method": "learned",
812
+ "metric": 0.5444630349091115,
813
+ "calls": 0.9222641295408784,
814
+ "gflops": 11923.244576085619,
815
+ "ms": 3441.363123586654
816
+ },
817
+ {
818
+ "pair": "grefcoco|instancevg|MiniCPM",
819
+ "method": "learned",
820
+ "metric": 0.539069279414684,
821
+ "calls": 0.9331140288687859,
822
+ "gflops": 12063.38347844168,
823
+ "ms": 3481.151581575278
824
+ },
825
+ {
826
+ "pair": "grefcoco|instancevg|MiniCPM",
827
+ "method": "learned",
828
+ "metric": 0.5336868672546294,
829
+ "calls": 0.9439185548592007,
830
+ "gflops": 12202.936332068595,
831
+ "ms": 3520.7736476852256
832
+ },
833
+ {
834
+ "pair": "grefcoco|instancevg|MiniCPM",
835
+ "method": "learned",
836
+ "metric": 0.5275557950259478,
837
+ "calls": 0.9553469642401383,
838
+ "gflops": 12350.547355721215,
839
+ "ms": 3562.683602126981
840
+ },
841
+ {
842
+ "pair": "grefcoco|instancevg|MiniCPM",
843
+ "method": "learned",
844
+ "metric": 0.520874571080169,
845
+ "calls": 0.966695970280464,
846
+ "gflops": 12497.132794097837,
847
+ "ms": 3604.3023707810516
848
+ },
849
+ {
850
+ "pair": "grefcoco|instancevg|MiniCPM",
851
+ "method": "learned",
852
+ "metric": 0.5142840938093752,
853
+ "calls": 0.977761392961461,
854
+ "gflops": 12640.05542791732,
855
+ "ms": 3644.8811901933914
856
+ },
857
+ {
858
+ "pair": "grefcoco|instancevg|MiniCPM",
859
+ "method": "learned",
860
+ "metric": 0.5079318265604174,
861
+ "calls": 0.9880441255707115,
862
+ "gflops": 12772.868721159106,
863
+ "ms": 3682.5897496985535
864
+ },
865
+ {
866
+ "pair": "grefcoco|instancevg|MiniCPM",
867
+ "method": "learned",
868
+ "metric": 0.5000708958398321,
869
+ "calls": 1.0,
870
+ "gflops": 12927.292561287999,
871
+ "ms": 3726.434009729928
872
+ },
873
+ {
874
+ "pair": "grefcoco|instancevg|MiniCPM",
875
+ "method": "confidence",
876
+ "metric": 0.6695119530385957,
877
+ "calls": 0.0,
878
+ "gflops": 7.502680064,
879
+ "ms": 46.08042127620138
880
+ },
881
+ {
882
+ "pair": "grefcoco|instancevg|MiniCPM",
883
+ "method": "confidence",
884
+ "metric": 0.6686044862887446,
885
+ "calls": 0.002467175226157729,
886
+ "gflops": 39.3690797110914,
887
+ "ms": 55.12797967926019
888
+ },
889
+ {
890
+ "pair": "grefcoco|instancevg|MiniCPM",
891
+ "method": "confidence",
892
+ "metric": 0.6678388112185577,
893
+ "calls": 0.004849275444516916,
894
+ "gflops": 70.13663799104171,
895
+ "ms": 63.86355330979973
896
+ },
897
+ {
898
+ "pair": "grefcoco|instancevg|MiniCPM",
899
+ "method": "confidence",
900
+ "metric": 0.667214927828035,
901
+ "calls": 0.007600034030003119,
902
+ "gflops": 105.66584219527005,
903
+ "ms": 73.95106095458944
904
+ },
905
+ {
906
+ "pair": "grefcoco|instancevg|MiniCPM",
907
+ "method": "confidence",
908
+ "metric": 0.6666761194453109,
909
+ "calls": 0.009811984232765221,
910
+ "gflops": 134.23571774093818,
911
+ "ms": 82.06266504009045
912
+ },
913
+ {
914
+ "pair": "grefcoco|instancevg|MiniCPM",
915
+ "method": "confidence",
916
+ "metric": 0.6662791027422511,
917
+ "calls": 0.012534384482318578,
918
+ "gflops": 169.39864148945287,
919
+ "ms": 92.04617776070705
920
+ },
921
+ {
922
+ "pair": "grefcoco|instancevg|MiniCPM",
923
+ "method": "confidence",
924
+ "metric": 0.6658537277032583,
925
+ "calls": 0.01607917647392451,
926
+ "gflops": 215.18369845366465,
927
+ "ms": 105.04554328234329
928
+ },
929
+ {
930
+ "pair": "grefcoco|instancevg|MiniCPM",
931
+ "method": "confidence",
932
+ "metric": 0.6654850693361314,
933
+ "calls": 0.019709043473328986,
934
+ "gflops": 262.0675967850175,
935
+ "ms": 118.35689357649878
936
+ },
937
+ {
938
+ "pair": "grefcoco|instancevg|MiniCPM",
939
+ "method": "confidence",
940
+ "metric": 0.6652865609846014,
941
+ "calls": 0.02353741882426339,
942
+ "gflops": 311.5154583063662,
943
+ "ms": 132.3962083398659
944
+ },
945
+ {
946
+ "pair": "grefcoco|instancevg|MiniCPM",
947
+ "method": "confidence",
948
+ "metric": 0.6648611859456087,
949
+ "calls": 0.028868785979638715,
950
+ "gflops": 380.3761839805407,
951
+ "ms": 151.94725408440678
952
+ },
953
+ {
954
+ "pair": "grefcoco|instancevg|MiniCPM",
955
+ "method": "confidence",
956
+ "metric": 0.6645208859144145,
957
+ "calls": 0.0345971698380739,
958
+ "gflops": 454.3648360347069,
959
+ "ms": 172.9542287673709
960
+ },
961
+ {
962
+ "pair": "grefcoco|instancevg|MiniCPM",
963
+ "method": "confidence",
964
+ "metric": 0.6642089442191532,
965
+ "calls": 0.03981510364971783,
966
+ "gflops": 521.7604398860266,
967
+ "ms": 192.08929481521943
968
+ },
969
+ {
970
+ "pair": "grefcoco|instancevg|MiniCPM",
971
+ "method": "confidence",
972
+ "metric": 0.6640387942035562,
973
+ "calls": 0.045855429203414344,
974
+ "gflops": 599.7781769530435,
975
+ "ms": 214.24021366408755
976
+ },
977
+ {
978
+ "pair": "grefcoco|instancevg|MiniCPM",
979
+ "method": "confidence",
980
+ "metric": 0.6637268525082948,
981
+ "calls": 0.05317187987408899,
982
+ "gflops": 694.2785345271766,
983
+ "ms": 241.0709041007447
984
+ },
985
+ {
986
+ "pair": "grefcoco|instancevg|MiniCPM",
987
+ "method": "confidence",
988
+ "metric": 0.6633581941411678,
989
+ "calls": 0.05960922213084536,
990
+ "gflops": 777.4241979741852,
991
+ "ms": 264.6777518880361
992
+ },
993
+ {
994
+ "pair": "grefcoco|instancevg|MiniCPM",
995
+ "method": "confidence",
996
+ "metric": 0.6630178941099736,
997
+ "calls": 0.0675211978561098,
998
+ "gflops": 879.6164451183058,
999
+ "ms": 293.69233573232816
1000
+ },
1001
+ {
1002
+ "pair": "grefcoco|instancevg|MiniCPM",
1003
+ "method": "confidence",
1004
+ "metric": 0.6626492357428466,
1005
+ "calls": 0.07551824858917279,
1006
+ "gflops": 982.9075336295676,
1007
+ "ms": 323.01890434913946
1008
+ },
1009
+ {
1010
+ "pair": "grefcoco|instancevg|MiniCPM",
1011
+ "method": "confidence",
1012
+ "metric": 0.6621671440319882,
1013
+ "calls": 0.08212574086152624,
1014
+ "gflops": 1068.2508798108586,
1015
+ "ms": 347.24972168146934
1016
+ },
1017
+ {
1018
+ "pair": "grefcoco|instancevg|MiniCPM",
1019
+ "method": "confidence",
1020
+ "metric": 0.6619969940163911,
1021
+ "calls": 0.08856308311828262,
1022
+ "gflops": 1151.396543257867,
1023
+ "ms": 370.85656946876077
1024
+ },
1025
+ {
1026
+ "pair": "grefcoco|instancevg|MiniCPM",
1027
+ "method": "confidence",
1028
+ "metric": 0.6615716189773984,
1029
+ "calls": 0.09480191702350906,
1030
+ "gflops": 1231.9782435148798,
1031
+ "ms": 393.7354527868405
1032
+ },
1033
+ {
1034
+ "pair": "grefcoco|instancevg|MiniCPM",
1035
+ "method": "confidence",
1036
+ "metric": 0.6612029606102714,
1037
+ "calls": 0.10007656750701868,
1038
+ "gflops": 1300.1064082776268,
1039
+ "ms": 413.0785086830352
1040
+ },
1041
+ {
1042
+ "pair": "grefcoco|instancevg|MiniCPM",
1043
+ "method": "confidence",
1044
+ "metric": 0.6608910189150101,
1045
+ "calls": 0.10554972634205825,
1046
+ "gflops": 1370.79853623037,
1047
+ "ms": 433.14952904844154
1048
+ },
1049
+ {
1050
+ "pair": "grefcoco|instancevg|MiniCPM",
1051
+ "method": "confidence",
1052
+ "metric": 0.6605790772197487,
1053
+ "calls": 0.11005870175538099,
1054
+ "gflops": 1429.0371286888474,
1055
+ "ms": 449.68472199196276
1056
+ },
1057
+ {
1058
+ "pair": "grefcoco|instancevg|MiniCPM",
1059
+ "method": "confidence",
1060
+ "metric": 0.6602671355244875,
1061
+ "calls": 0.11527663556702493,
1062
+ "gflops": 1496.432732540167,
1063
+ "ms": 468.8197880398113
1064
+ },
1065
+ {
1066
+ "pair": "grefcoco|instancevg|MiniCPM",
1067
+ "method": "confidence",
1068
+ "metric": 0.6599835521651589,
1069
+ "calls": 0.12029606102713893,
1070
+ "gflops": 1561.264373201491,
1071
+ "ms": 487.2268896184482
1072
+ },
1073
+ {
1074
+ "pair": "grefcoco|instancevg|MiniCPM",
1075
+ "method": "confidence",
1076
+ "metric": 0.6596999688058305,
1077
+ "calls": 0.12477667810452883,
1078
+ "gflops": 1619.1366852042547,
1079
+ "ms": 503.6580876377964
1080
+ },
1081
+ {
1082
+ "pair": "grefcoco|instancevg|MiniCPM",
1083
+ "method": "confidence",
1084
+ "metric": 0.659217877094972,
1085
+ "calls": 0.12914386183818732,
1086
+ "gflops": 1675.5438753841634,
1087
+ "ms": 519.6733059604521
1088
+ },
1089
+ {
1090
+ "pair": "grefcoco|instancevg|MiniCPM",
1091
+ "method": "confidence",
1092
+ "metric": 0.6587925020559794,
1093
+ "calls": 0.1322916371267334,
1094
+ "gflops": 1716.2010059683835,
1095
+ "ms": 531.2167425436651
1096
+ },
1097
+ {
1098
+ "pair": "grefcoco|instancevg|MiniCPM",
1099
+ "method": "confidence",
1100
+ "metric": 0.6582253353373224,
1101
+ "calls": 0.13626180415733205,
1102
+ "gflops": 1767.4802697683008,
1103
+ "ms": 545.7760319278977
1104
+ },
1105
+ {
1106
+ "pair": "grefcoco|instancevg|MiniCPM",
1107
+ "method": "confidence",
1108
+ "metric": 0.6578283186342625,
1109
+ "calls": 0.14023197118793068,
1110
+ "gflops": 1818.7595335682179,
1111
+ "ms": 560.3353213121303
1112
+ },
1113
+ {
1114
+ "pair": "grefcoco|instancevg|MiniCPM",
1115
+ "method": "confidence",
1116
+ "metric": 0.6573178685874713,
1117
+ "calls": 0.14326631313274538,
1118
+ "gflops": 1857.9515423295834,
1119
+ "ms": 571.462778198651
1120
+ },
1121
+ {
1122
+ "pair": "grefcoco|instancevg|MiniCPM",
1123
+ "method": "confidence",
1124
+ "metric": 0.6569208518844114,
1125
+ "calls": 0.14766185520233674,
1126
+ "gflops": 1914.725012965206,
1127
+ "ms": 587.5819914454798
1128
+ },
1129
+ {
1130
+ "pair": "grefcoco|instancevg|MiniCPM",
1131
+ "method": "confidence",
1132
+ "metric": 0.6565805518532173,
1133
+ "calls": 0.15066783881121856,
1134
+ "gflops": 1953.5507412708575,
1135
+ "ms": 598.6054534078274
1136
+ },
1137
+ {
1138
+ "pair": "grefcoco|instancevg|MiniCPM",
1139
+ "method": "confidence",
1140
+ "metric": 0.6561268184782917,
1141
+ "calls": 0.15452457249808582,
1142
+ "gflops": 2003.36488324792,
1143
+ "ms": 612.7487630953676
1144
+ },
1145
+ {
1146
+ "pair": "grefcoco|instancevg|MiniCPM",
1147
+ "method": "confidence",
1148
+ "metric": 0.6558715934548961,
1149
+ "calls": 0.15806936448969175,
1150
+ "gflops": 2049.1499402121317,
1151
+ "ms": 625.7481286170038
1152
+ },
1153
+ {
1154
+ "pair": "grefcoco|instancevg|MiniCPM",
1155
+ "method": "confidence",
1156
+ "metric": 0.6555596517596347,
1157
+ "calls": 0.16201117318435754,
1158
+ "gflops": 2100.0629235563347,
1159
+ "ms": 640.2034230770632
1160
+ },
1161
+ {
1162
+ "pair": "grefcoco|instancevg|MiniCPM",
1163
+ "method": "confidence",
1164
+ "metric": 0.6551626350565749,
1165
+ "calls": 0.1657544735274934,
1166
+ "gflops": 2148.4119437105423,
1167
+ "ms": 653.930753067911
1168
+ },
1169
+ {
1170
+ "pair": "grefcoco|instancevg|MiniCPM",
1171
+ "method": "confidence",
1172
+ "metric": 0.6545954683379179,
1173
+ "calls": 0.17012165726115192,
1174
+ "gflops": 2204.8191338904517,
1175
+ "ms": 669.9459713905669
1176
+ },
1177
+ {
1178
+ "pair": "grefcoco|instancevg|MiniCPM",
1179
+ "method": "confidence",
1180
+ "metric": 0.6540850182911266,
1181
+ "calls": 0.17536794940872868,
1182
+ "gflops": 2272.581018197485,
1183
+ "ms": 689.1850323625886
1184
+ },
1185
+ {
1186
+ "pair": "grefcoco|instancevg|MiniCPM",
1187
+ "method": "confidence",
1188
+ "metric": 0.6534044182287383,
1189
+ "calls": 0.17965005813458867,
1190
+ "gflops": 2327.889367010253,
1191
+ "ms": 704.8882659127252
1192
+ },
1193
+ {
1194
+ "pair": "grefcoco|instancevg|MiniCPM",
1195
+ "method": "confidence",
1196
+ "metric": 0.653092476533477,
1197
+ "calls": 0.18401724186824717,
1198
+ "gflops": 2384.2965571901614,
1199
+ "ms": 720.9034842353809
1200
+ },
1201
+ {
1202
+ "pair": "grefcoco|instancevg|MiniCPM",
1203
+ "method": "confidence",
1204
+ "metric": 0.6527521765022829,
1205
+ "calls": 0.18858293395343562,
1206
+ "gflops": 2443.2677105600665,
1207
+ "ms": 737.6466670272484
1208
+ },
1209
+ {
1210
+ "pair": "grefcoco|instancevg|MiniCPM",
1211
+ "method": "confidence",
1212
+ "metric": 0.6523835181351558,
1213
+ "calls": 0.19238295096843716,
1214
+ "gflops": 2492.3492916257014,
1215
+ "ms": 751.5819868664424
1216
+ },
1217
+ {
1218
+ "pair": "grefcoco|instancevg|MiniCPM",
1219
+ "method": "confidence",
1220
+ "metric": 0.6517596347446332,
1221
+ "calls": 0.19689192638175992,
1222
+ "gflops": 2550.5878840841788,
1223
+ "ms": 768.1171798099638
1224
+ },
1225
+ {
1226
+ "pair": "grefcoco|instancevg|MiniCPM",
1227
+ "method": "confidence",
1228
+ "metric": 0.6511641096900433,
1229
+ "calls": 0.20182627683407536,
1230
+ "gflops": 2614.3206833783615,
1231
+ "ms": 786.2122966160813
1232
+ },
1233
+ {
1234
+ "pair": "grefcoco|instancevg|MiniCPM",
1235
+ "method": "confidence",
1236
+ "metric": 0.6505402262995208,
1237
+ "calls": 0.20718600232538353,
1238
+ "gflops": 2683.5476895082497,
1239
+ "ms": 805.8673372847952
1240
+ },
1241
+ {
1242
+ "pair": "grefcoco|instancevg|MiniCPM",
1243
+ "method": "confidence",
1244
+ "metric": 0.6500864929245952,
1245
+ "calls": 0.21189348609023623,
1246
+ "gflops": 2744.350245156723,
1247
+ "ms": 823.1304946975282
1248
+ },
1249
+ {
1250
+ "pair": "grefcoco|instancevg|MiniCPM",
1251
+ "method": "confidence",
1252
+ "metric": 0.6500581345886624,
1253
+ "calls": 0.217139778237813,
1254
+ "gflops": 2812.1121294637564,
1255
+ "ms": 842.3695556695498
1256
+ },
1257
+ {
1258
+ "pair": "grefcoco|instancevg|MiniCPM",
1259
+ "method": "confidence",
1260
+ "metric": 0.6507670929869835,
1261
+ "calls": 0.22198905368232993,
1262
+ "gflops": 2874.7460873907985,
1263
+ "ms": 860.1526877031482
1264
+ },
1265
+ {
1266
+ "pair": "grefcoco|instancevg|MiniCPM",
1267
+ "method": "confidence",
1268
+ "metric": 0.6515611263931033,
1269
+ "calls": 0.22763236253296656,
1270
+ "gflops": 2947.635898077823,
1271
+ "ms": 880.8476776135931
1272
+ },
1273
+ {
1274
+ "pair": "grefcoco|instancevg|MiniCPM",
1275
+ "method": "confidence",
1276
+ "metric": 0.6527521765022829,
1277
+ "calls": 0.2345234381646485,
1278
+ "gflops": 3036.6420488162507,
1279
+ "ms": 906.1184441876538
1280
+ },
1281
+ {
1282
+ "pair": "grefcoco|instancevg|MiniCPM",
1283
+ "method": "confidence",
1284
+ "metric": 0.6537447182599325,
1285
+ "calls": 0.24382497235062248,
1286
+ "gflops": 3156.782038290343,
1287
+ "ms": 940.2287793164273
1288
+ },
1289
+ {
1290
+ "pair": "grefcoco|instancevg|MiniCPM",
1291
+ "method": "confidence",
1292
+ "metric": 0.6545387516660522,
1293
+ "calls": 0.252644414825738,
1294
+ "gflops": 3270.6952600173017,
1295
+ "ms": 972.5712007342581
1296
+ },
1297
+ {
1298
+ "pair": "grefcoco|instancevg|MiniCPM",
1299
+ "method": "confidence",
1300
+ "metric": 0.6538581516036639,
1301
+ "calls": 0.26401610753480986,
1302
+ "gflops": 3417.573722758493,
1303
+ "ms": 1014.2731653276672
1304
+ },
1305
+ {
1306
+ "pair": "grefcoco|instancevg|MiniCPM",
1307
+ "method": "confidence",
1308
+ "metric": 0.6537163599239997,
1309
+ "calls": 0.2760116836344043,
1310
+ "gflops": 3572.5103555253854,
1311
+ "ms": 1058.2630182528842
1312
+ },
1313
+ {
1314
+ "pair": "grefcoco|instancevg|MiniCPM",
1315
+ "method": "confidence",
1316
+ "metric": 0.6533477015568726,
1317
+ "calls": 0.2893401015228426,
1318
+ "gflops": 3744.6621697108217,
1319
+ "ms": 1107.1406326142362
1320
+ },
1321
+ {
1322
+ "pair": "grefcoco|instancevg|MiniCPM",
1323
+ "method": "confidence",
1324
+ "metric": 0.6525536681507529,
1325
+ "calls": 0.3036043444970649,
1326
+ "gflops": 3928.90123893481,
1327
+ "ms": 1159.4500794733005
1328
+ },
1329
+ {
1330
+ "pair": "grefcoco|instancevg|MiniCPM",
1331
+ "method": "confidence",
1332
+ "metric": 0.6509088846666478,
1333
+ "calls": 0.3175850041119587,
1334
+ "gflops": 4109.477503601662,
1335
+ "ms": 1210.719577090634
1336
+ },
1337
+ {
1338
+ "pair": "grefcoco|instancevg|MiniCPM",
1339
+ "method": "confidence",
1340
+ "metric": 0.6498312679011996,
1341
+ "calls": 0.33462836400760004,
1342
+ "gflops": 4329.612057485591,
1343
+ "ms": 1273.2205265186608
1344
+ },
1345
+ {
1346
+ "pair": "grefcoco|instancevg|MiniCPM",
1347
+ "method": "confidence",
1348
+ "metric": 0.6465984176048549,
1349
+ "calls": 0.35637920766809406,
1350
+ "gflops": 4610.549167017995,
1351
+ "ms": 1352.9846333594207
1352
+ },
1353
+ {
1354
+ "pair": "grefcoco|instancevg|MiniCPM",
1355
+ "method": "confidence",
1356
+ "metric": 0.6442730340583614,
1357
+ "calls": 0.37968975980489467,
1358
+ "gflops": 4911.631701614652,
1359
+ "ms": 1438.4684610297004
1360
+ },
1361
+ {
1362
+ "pair": "grefcoco|instancevg|MiniCPM",
1363
+ "method": "confidence",
1364
+ "metric": 0.6405580920511584,
1365
+ "calls": 0.4046450954258004,
1366
+ "gflops": 5233.958502642702,
1367
+ "ms": 1529.9839943020195
1368
+ },
1369
+ {
1370
+ "pair": "grefcoco|instancevg|MiniCPM",
1371
+ "method": "confidence",
1372
+ "metric": 0.6357371749425744,
1373
+ "calls": 0.43141536454640844,
1374
+ "gflops": 5579.72725283643,
1375
+ "ms": 1628.1552027214163
1376
+ },
1377
+ {
1378
+ "pair": "grefcoco|instancevg|MiniCPM",
1379
+ "method": "confidence",
1380
+ "metric": 0.630604316138729,
1381
+ "calls": 0.4610214672603012,
1382
+ "gflops": 5962.124048601527,
1383
+ "ms": 1736.725903558122
1384
+ },
1385
+ {
1386
+ "pair": "grefcoco|instancevg|MiniCPM",
1387
+ "method": "confidence",
1388
+ "metric": 0.6244221989053682,
1389
+ "calls": 0.4916484700677764,
1390
+ "gflops": 6357.706940772317,
1391
+ "ms": 1849.0404216650588
1392
+ },
1393
+ {
1394
+ "pair": "grefcoco|instancevg|MiniCPM",
1395
+ "method": "confidence",
1396
+ "metric": 0.6187788900547316,
1397
+ "calls": 0.5209426310864078,
1398
+ "gflops": 6736.074651524562,
1399
+ "ms": 1956.4671783358606
1400
+ },
1401
+ {
1402
+ "pair": "grefcoco|instancevg|MiniCPM",
1403
+ "method": "confidence",
1404
+ "metric": 0.6118310977511839,
1405
+ "calls": 0.5529308340186598,
1406
+ "gflops": 7149.2390055696105,
1407
+ "ms": 2073.773452803106
1408
+ },
1409
+ {
1410
+ "pair": "grefcoco|instancevg|MiniCPM",
1411
+ "method": "confidence",
1412
+ "metric": 0.6045146470805093,
1413
+ "calls": 0.5888041289737118,
1414
+ "gflops": 7612.583782047433,
1415
+ "ms": 2205.3270318820646
1416
+ },
1417
+ {
1418
+ "pair": "grefcoco|instancevg|MiniCPM",
1419
+ "method": "confidence",
1420
+ "metric": 0.5983608881830814,
1421
+ "calls": 0.6175027649377535,
1422
+ "gflops": 7983.2596032296915,
1423
+ "ms": 2310.5698951452314
1424
+ },
1425
+ {
1426
+ "pair": "grefcoco|instancevg|MiniCPM",
1427
+ "method": "confidence",
1428
+ "metric": 0.5925474293168477,
1429
+ "calls": 0.6421461588633979,
1430
+ "gflops": 8301.557319244892,
1431
+ "ms": 2400.941484251646
1432
+ },
1433
+ {
1434
+ "pair": "grefcoco|instancevg|MiniCPM",
1435
+ "method": "confidence",
1436
+ "metric": 0.5877548705441965,
1437
+ "calls": 0.665910444375124,
1438
+ "gflops": 8608.500341132967,
1439
+ "ms": 2488.0892307086956
1440
+ },
1441
+ {
1442
+ "pair": "grefcoco|instancevg|MiniCPM",
1443
+ "method": "confidence",
1444
+ "metric": 0.5829339534356124,
1445
+ "calls": 0.6838612710206166,
1446
+ "gflops": 8840.355869599738,
1447
+ "ms": 2553.9180177102617
1448
+ },
1449
+ {
1450
+ "pair": "grefcoco|instancevg|MiniCPM",
1451
+ "method": "confidence",
1452
+ "metric": 0.5779996029832969,
1453
+ "calls": 0.7029464311034228,
1454
+ "gflops": 9086.862616295053,
1455
+ "ms": 2623.9066016787506
1456
+ },
1457
+ {
1458
+ "pair": "grefcoco|instancevg|MiniCPM",
1459
+ "method": "confidence",
1460
+ "metric": 0.5745966026713553,
1461
+ "calls": 0.7180614241556306,
1462
+ "gflops": 9282.090099190453,
1463
+ "ms": 2679.3358962630077
1464
+ },
1465
+ {
1466
+ "pair": "grefcoco|instancevg|MiniCPM",
1467
+ "method": "confidence",
1468
+ "metric": 0.5710234523438165,
1469
+ "calls": 0.7352465757309361,
1470
+ "gflops": 9504.05605535295,
1471
+ "ms": 2742.3568203119
1472
+ },
1473
+ {
1474
+ "pair": "grefcoco|instancevg|MiniCPM",
1475
+ "method": "confidence",
1476
+ "metric": 0.567762243711539,
1477
+ "calls": 0.7478093185491875,
1478
+ "gflops": 9666.318297234116,
1479
+ "ms": 2788.426571720579
1480
+ },
1481
+ {
1482
+ "pair": "grefcoco|instancevg|MiniCPM",
1483
+ "method": "confidence",
1484
+ "metric": 0.565153276805717,
1485
+ "calls": 0.759804894648782,
1486
+ "gflops": 9821.254930001009,
1487
+ "ms": 2832.416424645796
1488
+ },
1489
+ {
1490
+ "pair": "grefcoco|instancevg|MiniCPM",
1491
+ "method": "confidence",
1492
+ "metric": 0.5620622181890367,
1493
+ "calls": 0.7729064458497575,
1494
+ "gflops": 9990.476500540737,
1495
+ "ms": 2880.4620796137633
1496
+ },
1497
+ {
1498
+ "pair": "grefcoco|instancevg|MiniCPM",
1499
+ "method": "confidence",
1500
+ "metric": 0.5588860845645578,
1501
+ "calls": 0.7855259053398747,
1502
+ "gflops": 10153.47130333333,
1503
+ "ms": 2926.739820870788
1504
+ },
1505
+ {
1506
+ "pair": "grefcoco|instancevg|MiniCPM",
1507
+ "method": "confidence",
1508
+ "metric": 0.5562771176587358,
1509
+ "calls": 0.7964722230099538,
1510
+ "gflops": 10294.855559238817,
1511
+ "ms": 2966.881861601601
1512
+ },
1513
+ {
1514
+ "pair": "grefcoco|instancevg|MiniCPM",
1515
+ "method": "confidence",
1516
+ "metric": 0.5536114340810481,
1517
+ "calls": 0.8082125740861527,
1518
+ "gflops": 10446.495667904286,
1519
+ "ms": 3009.93576020926
1520
+ },
1521
+ {
1522
+ "pair": "grefcoco|instancevg|MiniCPM",
1523
+ "method": "confidence",
1524
+ "metric": 0.5504920171284349,
1525
+ "calls": 0.8197544168108215,
1526
+ "gflops": 10595.57181337976,
1527
+ "ms": 3052.2616943477074
1528
+ },
1529
+ {
1530
+ "pair": "grefcoco|instancevg|MiniCPM",
1531
+ "method": "confidence",
1532
+ "metric": 0.5469188668008961,
1533
+ "calls": 0.8333380597226555,
1534
+ "gflops": 10771.02015166662,
1535
+ "ms": 3102.075263026618
1536
+ },
1537
+ {
1538
+ "pair": "grefcoco|instancevg|MiniCPM",
1539
+ "method": "confidence",
1540
+ "metric": 0.5450188582933954,
1541
+ "calls": 0.8426112355726966,
1542
+ "gflops": 10890.793860684997,
1543
+ "ms": 3136.0816032312177
1544
+ },
1545
+ {
1546
+ "pair": "grefcoco|instancevg|MiniCPM",
1547
+ "method": "confidence",
1548
+ "metric": 0.5423531747157077,
1549
+ "calls": 0.8529903865241187,
1550
+ "gflops": 11024.85250747621,
1551
+ "ms": 3174.1437454785687
1552
+ },
1553
+ {
1554
+ "pair": "grefcoco|instancevg|MiniCPM",
1555
+ "method": "confidence",
1556
+ "metric": 0.5395456994583557,
1557
+ "calls": 0.8635396874911381,
1558
+ "gflops": 11161.108837001704,
1559
+ "ms": 3212.829857270958
1560
+ },
1561
+ {
1562
+ "pair": "grefcoco|instancevg|MiniCPM",
1563
+ "method": "confidence",
1564
+ "metric": 0.5378725576383178,
1565
+ "calls": 0.8710262881774098,
1566
+ "gflops": 11257.806877310119,
1567
+ "ms": 3240.284517252654
1568
+ },
1569
+ {
1570
+ "pair": "grefcoco|instancevg|MiniCPM",
1571
+ "method": "confidence",
1572
+ "metric": 0.535774040779287,
1573
+ "calls": 0.8809517057539064,
1574
+ "gflops": 11386.005036809911,
1575
+ "ms": 3276.6827407132355
1576
+ },
1577
+ {
1578
+ "pair": "grefcoco|instancevg|MiniCPM",
1579
+ "method": "confidence",
1580
+ "metric": 0.5330516405297338,
1581
+ "calls": 0.8913024983693957,
1582
+ "gflops": 11519.69740314541,
1583
+ "ms": 3314.640888036413
1584
+ },
1585
+ {
1586
+ "pair": "grefcoco|instancevg|MiniCPM",
1587
+ "method": "confidence",
1588
+ "metric": 0.5304710319598446,
1589
+ "calls": 0.900915974250631,
1590
+ "gflops": 11643.866477632353,
1591
+ "ms": 3349.8951673310903
1592
+ },
1593
+ {
1594
+ "pair": "grefcoco|instancevg|MiniCPM",
1595
+ "method": "confidence",
1596
+ "metric": 0.5277486317102913,
1597
+ "calls": 0.910954825170859,
1598
+ "gflops": 11773.529758955001,
1599
+ "ms": 3386.7093704883646
1600
+ },
1601
+ {
1602
+ "pair": "grefcoco|instancevg|MiniCPM",
1603
+ "method": "confidence",
1604
+ "metric": 0.5245724980858123,
1605
+ "calls": 0.921901142840938,
1606
+ "gflops": 11914.914014860486,
1607
+ "ms": 3426.851411219177
1608
+ },
1609
+ {
1610
+ "pair": "grefcoco|instancevg|MiniCPM",
1611
+ "method": "confidence",
1612
+ "metric": 0.5217366644925275,
1613
+ "calls": 0.9303235686129938,
1614
+ "gflops": 12023.699310207454,
1615
+ "ms": 3457.737903698585
1616
+ },
1617
+ {
1618
+ "pair": "grefcoco|instancevg|MiniCPM",
1619
+ "method": "confidence",
1620
+ "metric": 0.5187306808836457,
1621
+ "calls": 0.9393698777755721,
1622
+ "gflops": 12140.542775580121,
1623
+ "ms": 3490.9122845098004
1624
+ },
1625
+ {
1626
+ "pair": "grefcoco|instancevg|MiniCPM",
1627
+ "method": "confidence",
1628
+ "metric": 0.515526188923234,
1629
+ "calls": 0.9490117119927403,
1630
+ "gflops": 12265.078130522777,
1631
+ "ms": 3526.270558728651
1632
+ },
1633
+ {
1634
+ "pair": "grefcoco|instancevg|MiniCPM",
1635
+ "method": "confidence",
1636
+ "metric": 0.5122366219550237,
1637
+ "calls": 0.9587953378895726,
1638
+ "gflops": 12391.444887744003,
1639
+ "ms": 3562.1488075683665
1640
+ },
1641
+ {
1642
+ "pair": "grefcoco|instancevg|MiniCPM",
1643
+ "method": "confidence",
1644
+ "metric": 0.5097694467288659,
1645
+ "calls": 0.9673595553412926,
1646
+ "gflops": 12502.061585369538,
1647
+ "ms": 3593.55527466864
1648
+ },
1649
+ {
1650
+ "pair": "grefcoco|instancevg|MiniCPM",
1651
+ "method": "confidence",
1652
+ "metric": 0.5075574965261038,
1653
+ "calls": 0.9746192893401016,
1654
+ "gflops": 12595.829382032243,
1655
+ "ms": 3620.177975256951
1656
+ },
1657
+ {
1658
+ "pair": "grefcoco|instancevg|MiniCPM",
1659
+ "method": "confidence",
1660
+ "metric": 0.5046365879250205,
1661
+ "calls": 0.9830984317840229,
1662
+ "gflops": 12705.347238290638,
1663
+ "ms": 3651.2724575847046
1664
+ },
1665
+ {
1666
+ "pair": "grefcoco|instancevg|MiniCPM",
1667
+ "method": "confidence",
1668
+ "metric": 0.5019425460114001,
1669
+ "calls": 0.9908118991577575,
1670
+ "gflops": 12804.975522244764,
1671
+ "ms": 3679.559076959785
1672
+ },
1673
+ {
1674
+ "pair": "grefcoco|instancevg|MiniCPM",
1675
+ "method": "confidence",
1676
+ "metric": 0.5000708958398321,
1677
+ "calls": 1.0,
1678
+ "gflops": 12923.650389896,
1679
+ "ms": 3713.253432391866
1680
+ },
1681
+ {
1682
+ "pair": "construction|yolo26x|Qwen3.8",
1683
+ "method": "learned",
1684
+ "metric": 0.18129624718318374,
1685
+ "calls": 0.0,
1686
+ "gflops": 139.16228659200002,
1687
+ "ms": 71.56936161300109
1688
+ },
1689
+ {
1690
+ "pair": "construction|yolo26x|Qwen3.8",
1691
+ "method": "learned",
1692
+ "metric": 0.18372855448704165,
1693
+ "calls": 0.007789613848202396,
1694
+ "gflops": 893.7788890248893,
1695
+ "ms": 148.08265569458447
1696
+ },
1697
+ {
1698
+ "pair": "construction|yolo26x|Qwen3.8",
1699
+ "method": "learned",
1700
+ "metric": 0.18685908224889786,
1701
+ "calls": 0.018908122503328895,
1702
+ "gflops": 1970.8812189932014,
1703
+ "ms": 257.29393869992145
1704
+ },
1705
+ {
1706
+ "pair": "construction|yolo26x|Qwen3.8",
1707
+ "method": "learned",
1708
+ "metric": 0.1884468119835047,
1709
+ "calls": 0.029227696404793613,
1710
+ "gflops": 2970.5869743530125,
1711
+ "ms": 358.6577043635576
1712
+ },
1713
+ {
1714
+ "pair": "construction|yolo26x|Qwen3.8",
1715
+ "method": "learned",
1716
+ "metric": 0.19016333859244455,
1717
+ "calls": 0.037150466045272966,
1718
+ "gflops": 3738.1030058873175,
1719
+ "ms": 436.47891800209106
1720
+ },
1721
+ {
1722
+ "pair": "construction|yolo26x|Qwen3.8",
1723
+ "method": "learned",
1724
+ "metric": 0.19203936879648048,
1725
+ "calls": 0.047203728362183756,
1726
+ "gflops": 4712.009903044295,
1727
+ "ms": 535.226844551827
1728
+ },
1729
+ {
1730
+ "pair": "construction|yolo26x|Qwen3.8",
1731
+ "method": "learned",
1732
+ "metric": 0.1973276972772499,
1733
+ "calls": 0.06245006657789613,
1734
+ "gflops": 6188.99453515653,
1735
+ "ms": 684.9836338226183
1736
+ },
1737
+ {
1738
+ "pair": "construction|yolo26x|Qwen3.8",
1739
+ "method": "learned",
1740
+ "metric": 0.19856778453770563,
1741
+ "calls": 0.07423435419440745,
1742
+ "gflops": 7330.594010631928,
1743
+ "ms": 800.734514612706
1744
+ },
1745
+ {
1746
+ "pair": "construction|yolo26x|Qwen3.8",
1747
+ "method": "learned",
1748
+ "metric": 0.19964444666304987,
1749
+ "calls": 0.08342210386151798,
1750
+ "gflops": 8220.654618629695,
1751
+ "ms": 890.980964042266
1752
+ },
1753
+ {
1754
+ "pair": "construction|yolo26x|Qwen3.8",
1755
+ "method": "learned",
1756
+ "metric": 0.2008447299915772,
1757
+ "calls": 0.09094540612516644,
1758
+ "gflops": 8949.47236285975,
1759
+ "ms": 964.878419009949
1760
+ },
1761
+ {
1762
+ "pair": "construction|yolo26x|Qwen3.8",
1763
+ "method": "learned",
1764
+ "metric": 0.2053760837500663,
1765
+ "calls": 0.10252996005326231,
1766
+ "gflops": 10071.722694683021,
1767
+ "ms": 1078.6674204646115
1768
+ },
1769
+ {
1770
+ "pair": "construction|yolo26x|Qwen3.8",
1771
+ "method": "learned",
1772
+ "metric": 0.2084699431938867,
1773
+ "calls": 0.11284953395472704,
1774
+ "gflops": 11071.428450042833,
1775
+ "ms": 1180.0311861282478
1776
+ },
1777
+ {
1778
+ "pair": "construction|yolo26x|Qwen3.8",
1779
+ "method": "learned",
1780
+ "metric": 0.21117730070554924,
1781
+ "calls": 0.12343541944074565,
1782
+ "gflops": 12096.933063605475,
1783
+ "ms": 1284.0107909057838
1784
+ },
1785
+ {
1786
+ "pair": "construction|yolo26x|Qwen3.8",
1787
+ "method": "learned",
1788
+ "metric": 0.21355089822347736,
1789
+ "calls": 0.1318908122503329,
1790
+ "gflops": 12916.04681154545,
1791
+ "ms": 1367.0636827721182
1792
+ },
1793
+ {
1794
+ "pair": "construction|yolo26x|Qwen3.8",
1795
+ "method": "learned",
1796
+ "metric": 0.21604151501630087,
1797
+ "calls": 0.14500665778961386,
1798
+ "gflops": 14186.640578035016,
1799
+ "ms": 1495.8937591317074
1800
+ },
1801
+ {
1802
+ "pair": "construction|yolo26x|Qwen3.8",
1803
+ "method": "learned",
1804
+ "metric": 0.21788898058414213,
1805
+ "calls": 0.1533288948069241,
1806
+ "gflops": 14992.854896873574,
1807
+ "ms": 1577.6387314410913
1808
+ },
1809
+ {
1810
+ "pair": "construction|yolo26x|Qwen3.8",
1811
+ "method": "learned",
1812
+ "metric": 0.2206453715306817,
1813
+ "calls": 0.16624500665778957,
1814
+ "gflops": 16244.09951971101,
1815
+ "ms": 1704.5069284652548
1816
+ },
1817
+ {
1818
+ "pair": "construction|yolo26x|Qwen3.8",
1819
+ "method": "learned",
1820
+ "metric": 0.22256047317491415,
1821
+ "calls": 0.17569906790945405,
1822
+ "gflops": 17159.958985911613,
1823
+ "ms": 1797.3692170087152
1824
+ },
1825
+ {
1826
+ "pair": "construction|yolo26x|Qwen3.8",
1827
+ "method": "learned",
1828
+ "metric": 0.22392279063908965,
1829
+ "calls": 0.18202396804260984,
1830
+ "gflops": 17772.681868228916,
1831
+ "ms": 1859.495395963847
1832
+ },
1833
+ {
1834
+ "pair": "construction|yolo26x|Qwen3.8",
1835
+ "method": "learned",
1836
+ "metric": 0.22639256671969005,
1837
+ "calls": 0.1914780292942743,
1838
+ "gflops": 18688.541334429516,
1839
+ "ms": 1952.3576845073073
1840
+ },
1841
+ {
1842
+ "pair": "construction|yolo26x|Qwen3.8",
1843
+ "method": "learned",
1844
+ "metric": 0.2303544739053073,
1845
+ "calls": 0.2051264980026631,
1846
+ "gflops": 20010.73281732475,
1847
+ "ms": 2086.419439094697
1848
+ },
1849
+ {
1850
+ "pair": "construction|yolo26x|Qwen3.8",
1851
+ "method": "learned",
1852
+ "metric": 0.2328940374950383,
1853
+ "calls": 0.2138482023968043,
1854
+ "gflops": 20855.645423467562,
1855
+ "ms": 2172.088170074932
1856
+ },
1857
+ {
1858
+ "pair": "construction|yolo26x|Qwen3.8",
1859
+ "method": "learned",
1860
+ "metric": 0.23741897056590364,
1861
+ "calls": 0.22656458055925435,
1862
+ "gflops": 22087.540902652876,
1863
+ "ms": 2296.9944877636703
1864
+ },
1865
+ {
1866
+ "pair": "construction|yolo26x|Qwen3.8",
1867
+ "method": "learned",
1868
+ "metric": 0.24026689104972293,
1869
+ "calls": 0.23794940079893476,
1870
+ "gflops": 23190.44209082402,
1871
+ "ms": 2408.8216098829075
1872
+ },
1873
+ {
1874
+ "pair": "construction|yolo26x|Qwen3.8",
1875
+ "method": "learned",
1876
+ "metric": 0.2431042580703525,
1877
+ "calls": 0.2474034620505992,
1878
+ "gflops": 24106.30155702462,
1879
+ "ms": 2501.6838984263677
1880
+ },
1881
+ {
1882
+ "pair": "construction|yolo26x|Qwen3.8",
1883
+ "method": "learned",
1884
+ "metric": 0.2446393126551028,
1885
+ "calls": 0.2549267643142477,
1886
+ "gflops": 24835.11930125468,
1887
+ "ms": 2575.581353394051
1888
+ },
1889
+ {
1890
+ "pair": "construction|yolo26x|Qwen3.8",
1891
+ "method": "learned",
1892
+ "metric": 0.24855675020919854,
1893
+ "calls": 0.26198402130492676,
1894
+ "gflops": 25518.78904362977,
1895
+ "ms": 2644.901089912408
1896
+ },
1897
+ {
1898
+ "pair": "construction|yolo26x|Qwen3.8",
1899
+ "method": "learned",
1900
+ "metric": 0.25234723459722813,
1901
+ "calls": 0.2728362183754993,
1902
+ "gflops": 26570.09251539525,
1903
+ "ms": 2751.4965338038446
1904
+ },
1905
+ {
1906
+ "pair": "construction|yolo26x|Qwen3.8",
1907
+ "method": "learned",
1908
+ "metric": 0.2555614615399192,
1909
+ "calls": 0.28169107856191744,
1910
+ "gflops": 27427.904550639476,
1911
+ "ms": 2838.4731843410295
1912
+ },
1913
+ {
1914
+ "pair": "construction|yolo26x|Qwen3.8",
1915
+ "method": "learned",
1916
+ "metric": 0.25818509905301673,
1917
+ "calls": 0.2905459387483355,
1918
+ "gflops": 28285.716585883696,
1919
+ "ms": 2925.4498348782136
1920
+ },
1921
+ {
1922
+ "pair": "construction|yolo26x|Qwen3.8",
1923
+ "method": "learned",
1924
+ "metric": 0.2606012166289425,
1925
+ "calls": 0.30099866844207723,
1926
+ "gflops": 29298.321770344926,
1927
+ "ms": 3028.1215200988004
1928
+ },
1929
+ {
1930
+ "pair": "construction|yolo26x|Qwen3.8",
1931
+ "method": "learned",
1932
+ "metric": 0.2653611798288512,
1933
+ "calls": 0.31178428761651134,
1934
+ "gflops": 30343.1755275597,
1935
+ "ms": 3134.063004211762
1936
+ },
1937
+ {
1938
+ "pair": "construction|yolo26x|Qwen3.8",
1939
+ "method": "learned",
1940
+ "metric": 0.26884911383364996,
1941
+ "calls": 0.3237017310252996,
1942
+ "gflops": 31497.674432136508,
1943
+ "ms": 3251.1218045588
1944
+ },
1945
+ {
1946
+ "pair": "construction|yolo26x|Qwen3.8",
1947
+ "method": "learned",
1948
+ "metric": 0.2729730863348169,
1949
+ "calls": 0.336884154460719,
1950
+ "gflops": 32774.717913176784,
1951
+ "ms": 3380.6058406968637
1952
+ },
1953
+ {
1954
+ "pair": "construction|yolo26x|Qwen3.8",
1955
+ "method": "learned",
1956
+ "metric": 0.27765278557626005,
1957
+ "calls": 0.34800266311584555,
1958
+ "gflops": 33851.8202431451,
1959
+ "ms": 3489.817123702201
1960
+ },
1961
+ {
1962
+ "pair": "construction|yolo26x|Qwen3.8",
1963
+ "method": "learned",
1964
+ "metric": 0.2796074774091736,
1965
+ "calls": 0.3575233022636485,
1966
+ "gflops": 34774.129423896404,
1967
+ "ms": 3583.333372024136
1968
+ },
1969
+ {
1970
+ "pair": "construction|yolo26x|Qwen3.8",
1971
+ "method": "learned",
1972
+ "metric": 0.28204715333825625,
1973
+ "calls": 0.3648468708388815,
1974
+ "gflops": 35483.59802447434,
1975
+ "ms": 3655.268947656394
1976
+ },
1977
+ {
1978
+ "pair": "construction|yolo26x|Qwen3.8",
1979
+ "method": "learned",
1980
+ "metric": 0.2865153302629788,
1981
+ "calls": 0.37490013315579224,
1982
+ "gflops": 36457.504921631305,
1983
+ "ms": 3754.0168742061296
1984
+ },
1985
+ {
1986
+ "pair": "construction|yolo26x|Qwen3.8",
1987
+ "method": "learned",
1988
+ "metric": 0.2895286620478625,
1989
+ "calls": 0.3859520639147802,
1990
+ "gflops": 37528.15753704891,
1991
+ "ms": 3862.574197432991
1992
+ },
1993
+ {
1994
+ "pair": "construction|yolo26x|Qwen3.8",
1995
+ "method": "learned",
1996
+ "metric": 0.2924523439388038,
1997
+ "calls": 0.39387483355525965,
1998
+ "gflops": 38295.673568583225,
1999
+ "ms": 3940.3954110715254
2000
+ },
2001
+ {
2002
+ "pair": "construction|yolo26x|Qwen3.8",
2003
+ "method": "learned",
2004
+ "metric": 0.29493220967499056,
2005
+ "calls": 0.403595206391478,
2006
+ "gflops": 39237.33189298665,
2007
+ "ms": 4035.8735387288853
2008
+ },
2009
+ {
2010
+ "pair": "construction|yolo26x|Qwen3.8",
2011
+ "method": "learned",
2012
+ "metric": 0.29679244853296216,
2013
+ "calls": 0.4097869507323568,
2014
+ "gflops": 39837.15534620254,
2015
+ "ms": 4096.691798127067
2016
+ },
2017
+ {
2018
+ "pair": "construction|yolo26x|Qwen3.8",
2019
+ "method": "learned",
2020
+ "metric": 0.29824032060818284,
2021
+ "calls": 0.4171105193075899,
2022
+ "gflops": 40546.62394678048,
2023
+ "ms": 4168.627373759326
2024
+ },
2025
+ {
2026
+ "pair": "construction|yolo26x|Qwen3.8",
2027
+ "method": "learned",
2028
+ "metric": 0.30060258544213647,
2029
+ "calls": 0.42676431424766975,
2030
+ "gflops": 41481.8325566332,
2031
+ "ms": 4263.45154163821
2032
+ },
2033
+ {
2034
+ "pair": "construction|yolo26x|Qwen3.8",
2035
+ "method": "learned",
2036
+ "metric": 0.30092311829652235,
2037
+ "calls": 0.4334886817576565,
2038
+ "gflops": 42133.253726254756,
2039
+ "ms": 4329.501479264193
2040
+ },
2041
+ {
2042
+ "pair": "construction|yolo26x|Qwen3.8",
2043
+ "method": "learned",
2044
+ "metric": 0.29908337970039717,
2045
+ "calls": 0.44560585885486015,
2046
+ "gflops": 43307.101774483686,
2047
+ "ms": 4448.5221589466555
2048
+ },
2049
+ {
2050
+ "pair": "construction|yolo26x|Qwen3.8",
2051
+ "method": "learned",
2052
+ "metric": 0.2977899223131896,
2053
+ "calls": 0.4533288948069242,
2054
+ "gflops": 44055.26866236588,
2055
+ "ms": 4524.381493249765
2056
+ },
2057
+ {
2058
+ "pair": "construction|yolo26x|Qwen3.8",
2059
+ "method": "learned",
2060
+ "metric": 0.2982768381293317,
2061
+ "calls": 0.45952063914780295,
2062
+ "gflops": 44655.09211558176,
2063
+ "ms": 4585.199752647946
2064
+ },
2065
+ {
2066
+ "pair": "construction|yolo26x|Qwen3.8",
2067
+ "method": "learned",
2068
+ "metric": 0.29713930300206126,
2069
+ "calls": 0.46677762982689747,
2070
+ "gflops": 45358.11100160898,
2071
+ "ms": 4656.481368501729
2072
+ },
2073
+ {
2074
+ "pair": "construction|yolo26x|Qwen3.8",
2075
+ "method": "learned",
2076
+ "metric": 0.2981419006299502,
2077
+ "calls": 0.4787616511318243,
2078
+ "gflops": 46519.05962073651,
2079
+ "ms": 4774.194128627242
2080
+ },
2081
+ {
2082
+ "pair": "construction|yolo26x|Qwen3.8",
2083
+ "method": "learned",
2084
+ "metric": 0.30134517763453006,
2085
+ "calls": 0.4900133155792277,
2086
+ "gflops": 47609.06137980623,
2087
+ "ms": 4884.7133311895295
2088
+ },
2089
+ {
2090
+ "pair": "construction|yolo26x|Qwen3.8",
2091
+ "method": "learned",
2092
+ "metric": 0.3013386133731364,
2093
+ "calls": 0.5019973368841545,
2094
+ "gflops": 48770.009998933754,
2095
+ "ms": 5002.426091315042
2096
+ },
2097
+ {
2098
+ "pair": "construction|yolo26x|Qwen3.8",
2099
+ "method": "learned",
2100
+ "metric": 0.3006032949818893,
2101
+ "calls": 0.5176431424766977,
2102
+ "gflops": 50285.69291835024,
2103
+ "ms": 5156.106639256684
2104
+ },
2105
+ {
2106
+ "pair": "construction|yolo26x|Qwen3.8",
2107
+ "method": "learned",
2108
+ "metric": 0.29926650757486717,
2109
+ "calls": 0.5346205059920106,
2110
+ "gflops": 51930.370128780894,
2111
+ "ms": 5322.8663827678265
2112
+ },
2113
+ {
2114
+ "pair": "construction|yolo26x|Qwen3.8",
2115
+ "method": "learned",
2116
+ "metric": 0.29911586063303675,
2117
+ "calls": 0.5474700399467377,
2118
+ "gflops": 53175.16503706763,
2119
+ "ms": 5449.080620013516
2120
+ },
2121
+ {
2122
+ "pair": "construction|yolo26x|Qwen3.8",
2123
+ "method": "learned",
2124
+ "metric": 0.29813893282180814,
2125
+ "calls": 0.5575898801597869,
2126
+ "gflops": 54155.52164877531,
2127
+ "ms": 5548.482506341727
2128
+ },
2129
+ {
2130
+ "pair": "construction|yolo26x|Qwen3.8",
2131
+ "method": "learned",
2132
+ "metric": 0.2974509950363013,
2133
+ "calls": 0.5675765645805592,
2134
+ "gflops": 55122.97883138157,
2135
+ "ms": 5646.576473112987
2136
+ },
2137
+ {
2138
+ "pair": "construction|yolo26x|Qwen3.8",
2139
+ "method": "learned",
2140
+ "metric": 0.29724357969129245,
2141
+ "calls": 0.5804260985352863,
2142
+ "gflops": 56367.773739668315,
2143
+ "ms": 5772.790710358676
2144
+ },
2145
+ {
2146
+ "pair": "construction|yolo26x|Qwen3.8",
2147
+ "method": "learned",
2148
+ "metric": 0.29665775151403084,
2149
+ "calls": 0.5938082556591212,
2150
+ "gflops": 57664.16636436072,
2151
+ "ms": 5904.236625832166
2152
+ },
2153
+ {
2154
+ "pair": "construction|yolo26x|Qwen3.8",
2155
+ "method": "learned",
2156
+ "metric": 0.2975661409457673,
2157
+ "calls": 0.603595206391478,
2158
+ "gflops": 58612.274403314856,
2159
+ "ms": 6000.368713268001
2160
+ },
2161
+ {
2162
+ "pair": "construction|yolo26x|Qwen3.8",
2163
+ "method": "learned",
2164
+ "metric": 0.29631335243812956,
2165
+ "calls": 0.6135153129161118,
2166
+ "gflops": 59573.28187137042,
2167
+ "ms": 6097.808720260787
2168
+ },
2169
+ {
2170
+ "pair": "construction|yolo26x|Qwen3.8",
2171
+ "method": "learned",
2172
+ "metric": 0.2960348088956203,
2173
+ "calls": 0.6214380825565912,
2174
+ "gflops": 60340.79790290472,
2175
+ "ms": 6175.62993389932
2176
+ },
2177
+ {
2178
+ "pair": "construction|yolo26x|Qwen3.8",
2179
+ "method": "learned",
2180
+ "metric": 0.2951989613231927,
2181
+ "calls": 0.6318242343541943,
2182
+ "gflops": 61346.95337281523,
2183
+ "ms": 6277.647659341431
2184
+ },
2185
+ {
2186
+ "pair": "construction|yolo26x|Qwen3.8",
2187
+ "method": "learned",
2188
+ "metric": 0.29386485972911613,
2189
+ "calls": 0.6431424766977363,
2190
+ "gflops": 62443.404846435675,
2191
+ "ms": 6388.820821682193
2192
+ },
2193
+ {
2194
+ "pair": "construction|yolo26x|Qwen3.8",
2195
+ "method": "learned",
2196
+ "metric": 0.292479005789999,
2197
+ "calls": 0.6534620505992011,
2198
+ "gflops": 63443.110601795495,
2199
+ "ms": 6490.184587345831
2200
+ },
2201
+ {
2202
+ "pair": "construction|yolo26x|Qwen3.8",
2203
+ "method": "learned",
2204
+ "metric": 0.2926539927366717,
2205
+ "calls": 0.6655126498002664,
2206
+ "gflops": 64610.50893547373,
2207
+ "ms": 6608.551307249818
2208
+ },
2209
+ {
2210
+ "pair": "construction|yolo26x|Qwen3.8",
2211
+ "method": "learned",
2212
+ "metric": 0.29278795045342987,
2213
+ "calls": 0.6742343541944075,
2214
+ "gflops": 65455.42154161653,
2215
+ "ms": 6694.2200382300525
2216
+ },
2217
+ {
2218
+ "pair": "construction|yolo26x|Qwen3.8",
2219
+ "method": "learned",
2220
+ "metric": 0.29144368638516527,
2221
+ "calls": 0.6844207723035952,
2222
+ "gflops": 66442.22786787493,
2223
+ "ms": 6794.275884336738
2224
+ },
2225
+ {
2226
+ "pair": "construction|yolo26x|Qwen3.8",
2227
+ "method": "learned",
2228
+ "metric": 0.2913839507485529,
2229
+ "calls": 0.6950732356857523,
2230
+ "gflops": 67474.18219598828,
2231
+ "ms": 6898.909448892749
2232
+ },
2233
+ {
2234
+ "pair": "construction|yolo26x|Qwen3.8",
2235
+ "method": "learned",
2236
+ "metric": 0.2916685120367808,
2237
+ "calls": 0.70219707057257,
2238
+ "gflops": 68164.30165291409,
2239
+ "ms": 6968.883145189583
2240
+ },
2241
+ {
2242
+ "pair": "construction|yolo26x|Qwen3.8",
2243
+ "method": "learned",
2244
+ "metric": 0.2908903102338221,
2245
+ "calls": 0.70745672436751,
2246
+ "gflops": 68673.82910242004,
2247
+ "ms": 7020.545967689112
2248
+ },
2249
+ {
2250
+ "pair": "construction|yolo26x|Qwen3.8",
2251
+ "method": "learned",
2252
+ "metric": 0.29068952549932253,
2253
+ "calls": 0.7203728362183754,
2254
+ "gflops": 69925.07372525749,
2255
+ "ms": 7147.4141647132765
2256
+ },
2257
+ {
2258
+ "pair": "construction|yolo26x|Qwen3.8",
2259
+ "method": "learned",
2260
+ "metric": 0.28982721462710204,
2261
+ "calls": 0.7292942743009322,
2262
+ "gflops": 70789.33547505244,
2263
+ "ms": 7235.044775028938
2264
+ },
2265
+ {
2266
+ "pair": "construction|yolo26x|Qwen3.8",
2267
+ "method": "learned",
2268
+ "metric": 0.289887284031958,
2269
+ "calls": 0.7372836218375498,
2270
+ "gflops": 71563.30122113743,
2271
+ "ms": 7313.519948445944
2272
+ },
2273
+ {
2274
+ "pair": "construction|yolo26x|Qwen3.8",
2275
+ "method": "learned",
2276
+ "metric": 0.2914457139433469,
2277
+ "calls": 0.7460719041278296,
2278
+ "gflops": 72414.66354183097,
2279
+ "ms": 7399.842639204656
2280
+ },
2281
+ {
2282
+ "pair": "construction|yolo26x|Qwen3.8",
2283
+ "method": "learned",
2284
+ "metric": 0.29113442803741185,
2285
+ "calls": 0.7552596537949402,
2286
+ "gflops": 73304.72414982873,
2287
+ "ms": 7490.089088634216
2288
+ },
2289
+ {
2290
+ "pair": "construction|yolo26x|Qwen3.8",
2291
+ "method": "learned",
2292
+ "metric": 0.29031315276962183,
2293
+ "calls": 0.7674434087882822,
2294
+ "gflops": 74485.02191260837,
2295
+ "ms": 7609.763728095152
2296
+ },
2297
+ {
2298
+ "pair": "construction|yolo26x|Qwen3.8",
2299
+ "method": "learned",
2300
+ "metric": 0.289990968297447,
2301
+ "calls": 0.7755659121171771,
2302
+ "gflops": 75271.8870877948,
2303
+ "ms": 7689.5468210691115
2304
+ },
2305
+ {
2306
+ "pair": "construction|yolo26x|Qwen3.8",
2307
+ "method": "learned",
2308
+ "metric": 0.2901920651624007,
2309
+ "calls": 0.7833555259653795,
2310
+ "gflops": 76026.50369022769,
2311
+ "ms": 7766.060115150694
2312
+ },
2313
+ {
2314
+ "pair": "construction|yolo26x|Qwen3.8",
2315
+ "method": "learned",
2316
+ "metric": 0.2901181158873772,
2317
+ "calls": 0.7904793608521972,
2318
+ "gflops": 76716.62314715351,
2319
+ "ms": 7836.033811447528
2320
+ },
2321
+ {
2322
+ "pair": "construction|yolo26x|Qwen3.8",
2323
+ "method": "learned",
2324
+ "metric": 0.28973901353907683,
2325
+ "calls": 0.7987350199733687,
2326
+ "gflops": 77516.38775144133,
2327
+ "ms": 7917.124823978435
2328
+ },
2329
+ {
2330
+ "pair": "construction|yolo26x|Qwen3.8",
2331
+ "method": "learned",
2332
+ "metric": 0.28888237446044834,
2333
+ "calls": 0.8127829560585885,
2334
+ "gflops": 78877.27752164083,
2335
+ "ms": 8055.110337236676
2336
+ },
2337
+ {
2338
+ "pair": "construction|yolo26x|Qwen3.8",
2339
+ "method": "learned",
2340
+ "metric": 0.2890360783246395,
2341
+ "calls": 0.8225699067909453,
2342
+ "gflops": 79825.38556059496,
2343
+ "ms": 8151.242424672511
2344
+ },
2345
+ {
2346
+ "pair": "construction|yolo26x|Qwen3.8",
2347
+ "method": "learned",
2348
+ "metric": 0.28767900194586976,
2349
+ "calls": 0.8346205059920105,
2350
+ "gflops": 80992.78389427319,
2351
+ "ms": 8269.6091445765
2352
+ },
2353
+ {
2354
+ "pair": "construction|yolo26x|Qwen3.8",
2355
+ "method": "learned",
2356
+ "metric": 0.28869706841582,
2357
+ "calls": 0.8463382157123837,
2358
+ "gflops": 82127.9336551979,
2359
+ "ms": 8384.706065588114
2360
+ },
2361
+ {
2362
+ "pair": "construction|yolo26x|Qwen3.8",
2363
+ "method": "learned",
2364
+ "metric": 0.2875418385011496,
2365
+ "calls": 0.8567243675099867,
2366
+ "gflops": 83134.08912510841,
2367
+ "ms": 8486.723791030225
2368
+ },
2369
+ {
2370
+ "pair": "construction|yolo26x|Qwen3.8",
2371
+ "method": "learned",
2372
+ "metric": 0.28776751513017684,
2373
+ "calls": 0.86491344873502,
2374
+ "gflops": 83927.40401484555,
2375
+ "ms": 8567.160843782658
2376
+ },
2377
+ {
2378
+ "pair": "construction|yolo26x|Qwen3.8",
2379
+ "method": "learned",
2380
+ "metric": 0.2880879394318149,
2381
+ "calls": 0.8767643142476697,
2382
+ "gflops": 85075.45320487165,
2383
+ "ms": 8683.565684351219
2384
+ },
2385
+ {
2386
+ "pair": "construction|yolo26x|Qwen3.8",
2387
+ "method": "learned",
2388
+ "metric": 0.287273877029594,
2389
+ "calls": 0.8856857523302264,
2390
+ "gflops": 85939.71495466659,
2391
+ "ms": 8771.196294666881
2392
+ },
2393
+ {
2394
+ "pair": "construction|yolo26x|Qwen3.8",
2395
+ "method": "learned",
2396
+ "metric": 0.28547632945289336,
2397
+ "calls": 0.8951398135818909,
2398
+ "gflops": 86855.5744208672,
2399
+ "ms": 8864.05858321034
2400
+ },
2401
+ {
2402
+ "pair": "construction|yolo26x|Qwen3.8",
2403
+ "method": "learned",
2404
+ "metric": 0.284269379519687,
2405
+ "calls": 0.9029960053262316,
2406
+ "gflops": 87616.64073785077,
2407
+ "ms": 8941.225837070398
2408
+ },
2409
+ {
2410
+ "pair": "construction|yolo26x|Qwen3.8",
2411
+ "method": "learned",
2412
+ "metric": 0.28250852235623786,
2413
+ "calls": 0.9131824234354193,
2414
+ "gflops": 88603.44706410918,
2415
+ "ms": 9041.281683177083
2416
+ },
2417
+ {
2418
+ "pair": "construction|yolo26x|Qwen3.8",
2419
+ "method": "learned",
2420
+ "metric": 0.2814829677111451,
2421
+ "calls": 0.9243675099866845,
2422
+ "gflops": 89686.99910862821,
2423
+ "ms": 9151.146925960897
2424
+ },
2425
+ {
2426
+ "pair": "construction|yolo26x|Qwen3.8",
2427
+ "method": "learned",
2428
+ "metric": 0.2800886018148173,
2429
+ "calls": 0.9328229027962717,
2430
+ "gflops": 90506.11285656819,
2431
+ "ms": 9234.199817827232
2432
+ },
2433
+ {
2434
+ "pair": "construction|yolo26x|Qwen3.8",
2435
+ "method": "learned",
2436
+ "metric": 0.27854520316604037,
2437
+ "calls": 0.9400798934753661,
2438
+ "gflops": 91209.13174259539,
2439
+ "ms": 9305.481433681012
2440
+ },
2441
+ {
2442
+ "pair": "construction|yolo26x|Qwen3.8",
2443
+ "method": "learned",
2444
+ "metric": 0.27746458817138653,
2445
+ "calls": 0.9482689747003995,
2446
+ "gflops": 92002.44663233255,
2447
+ "ms": 9385.918486433447
2448
+ },
2449
+ {
2450
+ "pair": "construction|yolo26x|Qwen3.8",
2451
+ "method": "learned",
2452
+ "metric": 0.2760878894837333,
2453
+ "calls": 0.9603861517976032,
2454
+ "gflops": 93176.29468056148,
2455
+ "ms": 9504.939166115912
2456
+ },
2457
+ {
2458
+ "pair": "construction|yolo26x|Qwen3.8",
2459
+ "method": "learned",
2460
+ "metric": 0.27468746115763226,
2461
+ "calls": 0.9677762982689748,
2462
+ "gflops": 93892.21299569013,
2463
+ "ms": 9577.528701526644
2464
+ },
2465
+ {
2466
+ "pair": "construction|yolo26x|Qwen3.8",
2467
+ "method": "learned",
2468
+ "metric": 0.2732948963509506,
2469
+ "calls": 0.9782290279627164,
2470
+ "gflops": 94904.81818015134,
2471
+ "ms": 9680.20038674723
2472
+ },
2473
+ {
2474
+ "pair": "construction|yolo26x|Qwen3.8",
2475
+ "method": "learned",
2476
+ "metric": 0.27348968306795485,
2477
+ "calls": 0.9849533954727031,
2478
+ "gflops": 95556.2393497729,
2479
+ "ms": 9746.250324373212
2480
+ },
2481
+ {
2482
+ "pair": "construction|yolo26x|Qwen3.8",
2483
+ "method": "learned",
2484
+ "metric": 0.27001009067886245,
2485
+ "calls": 1.0,
2486
+ "gflops": 97013.87483823301,
2487
+ "ms": 9894.045234308578
2488
+ },
2489
+ {
2490
+ "pair": "construction|yolo26x|Qwen3.8",
2491
+ "method": "confidence",
2492
+ "metric": 0.18129624718318377,
2493
+ "calls": 0.0,
2494
+ "gflops": 135.97853184000002,
2495
+ "ms": 46.30809425361804
2496
+ },
2497
+ {
2498
+ "pair": "construction|yolo26x|Qwen3.8",
2499
+ "method": "confidence",
2500
+ "metric": 0.18129624718318377,
2501
+ "calls": 0.0,
2502
+ "gflops": 135.97853184000002,
2503
+ "ms": 46.30809425361804
2504
+ },
2505
+ {
2506
+ "pair": "construction|yolo26x|Qwen3.8",
2507
+ "method": "confidence",
2508
+ "metric": 0.18129624718318377,
2509
+ "calls": 0.0,
2510
+ "gflops": 135.97853184000002,
2511
+ "ms": 46.30809425361804
2512
+ },
2513
+ {
2514
+ "pair": "construction|yolo26x|Qwen3.8",
2515
+ "method": "confidence",
2516
+ "metric": 0.18129624718318377,
2517
+ "calls": 0.0,
2518
+ "gflops": 135.97853184000002,
2519
+ "ms": 46.30809425361804
2520
+ },
2521
+ {
2522
+ "pair": "construction|yolo26x|Qwen3.8",
2523
+ "method": "confidence",
2524
+ "metric": 0.18129624718318377,
2525
+ "calls": 0.0,
2526
+ "gflops": 135.97853184000002,
2527
+ "ms": 46.30809425361804
2528
+ },
2529
+ {
2530
+ "pair": "construction|yolo26x|Qwen3.8",
2531
+ "method": "confidence",
2532
+ "metric": 0.18129624718318377,
2533
+ "calls": 0.0,
2534
+ "gflops": 135.97853184000002,
2535
+ "ms": 46.30809425361804
2536
+ },
2537
+ {
2538
+ "pair": "construction|yolo26x|Qwen3.8",
2539
+ "method": "confidence",
2540
+ "metric": 0.18129624718318377,
2541
+ "calls": 0.0,
2542
+ "gflops": 135.97853184000002,
2543
+ "ms": 46.30809425361804
2544
+ },
2545
+ {
2546
+ "pair": "construction|yolo26x|Qwen3.8",
2547
+ "method": "confidence",
2548
+ "metric": 0.18129624718318377,
2549
+ "calls": 0.0,
2550
+ "gflops": 135.97853184000002,
2551
+ "ms": 46.30809425361804
2552
+ },
2553
+ {
2554
+ "pair": "construction|yolo26x|Qwen3.8",
2555
+ "method": "confidence",
2556
+ "metric": 0.18129624718318377,
2557
+ "calls": 0.0,
2558
+ "gflops": 135.97853184000002,
2559
+ "ms": 46.30809425361804
2560
+ },
2561
+ {
2562
+ "pair": "construction|yolo26x|Qwen3.8",
2563
+ "method": "confidence",
2564
+ "metric": 0.18129624718318377,
2565
+ "calls": 0.0,
2566
+ "gflops": 135.97853184000002,
2567
+ "ms": 46.30809425361804
2568
+ },
2569
+ {
2570
+ "pair": "construction|yolo26x|Qwen3.8",
2571
+ "method": "confidence",
2572
+ "metric": 0.18129624718318377,
2573
+ "calls": 0.0,
2574
+ "gflops": 135.97853184000002,
2575
+ "ms": 46.30809425361804
2576
+ },
2577
+ {
2578
+ "pair": "construction|yolo26x|Qwen3.8",
2579
+ "method": "confidence",
2580
+ "metric": 0.18129624718318377,
2581
+ "calls": 0.0,
2582
+ "gflops": 135.97853184000002,
2583
+ "ms": 46.30809425361804
2584
+ },
2585
+ {
2586
+ "pair": "construction|yolo26x|Qwen3.8",
2587
+ "method": "confidence",
2588
+ "metric": 0.18129624718318377,
2589
+ "calls": 0.0,
2590
+ "gflops": 135.97853184000002,
2591
+ "ms": 46.30809425361804
2592
+ },
2593
+ {
2594
+ "pair": "construction|yolo26x|Qwen3.8",
2595
+ "method": "confidence",
2596
+ "metric": 0.18129624718318377,
2597
+ "calls": 0.0,
2598
+ "gflops": 135.97853184000002,
2599
+ "ms": 46.30809425361804
2600
+ },
2601
+ {
2602
+ "pair": "construction|yolo26x|Qwen3.8",
2603
+ "method": "confidence",
2604
+ "metric": 0.18129624718318377,
2605
+ "calls": 0.0,
2606
+ "gflops": 135.97853184000002,
2607
+ "ms": 46.30809425361804
2608
+ },
2609
+ {
2610
+ "pair": "construction|yolo26x|Qwen3.8",
2611
+ "method": "confidence",
2612
+ "metric": 0.18129624718318377,
2613
+ "calls": 0.0,
2614
+ "gflops": 135.97853184000002,
2615
+ "ms": 46.30809425361804
2616
+ },
2617
+ {
2618
+ "pair": "construction|yolo26x|Qwen3.8",
2619
+ "method": "confidence",
2620
+ "metric": 0.18129624718318377,
2621
+ "calls": 0.0,
2622
+ "gflops": 135.97853184000002,
2623
+ "ms": 46.30809425361804
2624
+ },
2625
+ {
2626
+ "pair": "construction|yolo26x|Qwen3.8",
2627
+ "method": "confidence",
2628
+ "metric": 0.18129624718318377,
2629
+ "calls": 0.0,
2630
+ "gflops": 135.97853184000002,
2631
+ "ms": 46.30809425361804
2632
+ },
2633
+ {
2634
+ "pair": "construction|yolo26x|Qwen3.8",
2635
+ "method": "confidence",
2636
+ "metric": 0.18129624718318377,
2637
+ "calls": 0.0,
2638
+ "gflops": 135.97853184000002,
2639
+ "ms": 46.30809425361804
2640
+ },
2641
+ {
2642
+ "pair": "construction|yolo26x|Qwen3.8",
2643
+ "method": "confidence",
2644
+ "metric": 0.1784481207469488,
2645
+ "calls": 0.17942743009320905,
2646
+ "gflops": 17517.959245999285,
2647
+ "ms": 1808.7296972439362
2648
+ },
2649
+ {
2650
+ "pair": "construction|yolo26x|Qwen3.8",
2651
+ "method": "confidence",
2652
+ "metric": 0.1780928651788194,
2653
+ "calls": 0.19340878828229027,
2654
+ "gflops": 18872.39930164806,
2655
+ "ms": 1946.0612507237013
2656
+ },
2657
+ {
2658
+ "pair": "construction|yolo26x|Qwen3.8",
2659
+ "method": "confidence",
2660
+ "metric": 0.17829989216687975,
2661
+ "calls": 0.20306258322237017,
2662
+ "gflops": 19807.60791150079,
2663
+ "ms": 2040.8854186025867
2664
+ },
2665
+ {
2666
+ "pair": "construction|yolo26x|Qwen3.8",
2667
+ "method": "confidence",
2668
+ "metric": 0.1779854683890517,
2669
+ "calls": 0.20838881491344874,
2670
+ "gflops": 20323.585075557465,
2671
+ "ms": 2093.2022008805925
2672
+ },
2673
+ {
2674
+ "pair": "construction|yolo26x|Qwen3.8",
2675
+ "method": "confidence",
2676
+ "metric": 0.17917657728013692,
2677
+ "calls": 0.21537949400798934,
2678
+ "gflops": 21000.805103381852,
2679
+ "ms": 2161.867977620475
2680
+ },
2681
+ {
2682
+ "pair": "construction|yolo26x|Qwen3.8",
2683
+ "method": "confidence",
2684
+ "metric": 0.17803687432776102,
2685
+ "calls": 0.23135818908122505,
2686
+ "gflops": 22548.736595551884,
2687
+ "ms": 2318.8183244544925
2688
+ },
2689
+ {
2690
+ "pair": "construction|yolo26x|Qwen3.8",
2691
+ "method": "confidence",
2692
+ "metric": 0.18026802885144627,
2693
+ "calls": 0.24800266311584554,
2694
+ "gflops": 24161.165233228996,
2695
+ "ms": 2482.3082690732604
2696
+ },
2697
+ {
2698
+ "pair": "construction|yolo26x|Qwen3.8",
2699
+ "method": "confidence",
2700
+ "metric": 0.18119827871024874,
2701
+ "calls": 0.2589880159786951,
2702
+ "gflops": 25225.368134095894,
2703
+ "ms": 2590.211632521647
2704
+ },
2705
+ {
2706
+ "pair": "construction|yolo26x|Qwen3.8",
2707
+ "method": "confidence",
2708
+ "metric": 0.17963665085316138,
2709
+ "calls": 0.27430093209054596,
2710
+ "gflops": 26708.802480758837,
2711
+ "ms": 2740.6223815709136
2712
+ },
2713
+ {
2714
+ "pair": "construction|yolo26x|Qwen3.8",
2715
+ "method": "confidence",
2716
+ "metric": 0.17924095839307458,
2717
+ "calls": 0.281624500665779,
2718
+ "gflops": 27418.271081336767,
2719
+ "ms": 2812.5579572031716
2720
+ },
2721
+ {
2722
+ "pair": "construction|yolo26x|Qwen3.8",
2723
+ "method": "confidence",
2724
+ "metric": 0.1789157174618718,
2725
+ "calls": 0.28761651131824234,
2726
+ "gflops": 27998.745390900527,
2727
+ "ms": 2871.4143372659278
2728
+ },
2729
+ {
2730
+ "pair": "construction|yolo26x|Qwen3.8",
2731
+ "method": "confidence",
2732
+ "metric": 0.1787206845800119,
2733
+ "calls": 0.2969374167776298,
2734
+ "gflops": 28901.70542799971,
2735
+ "ms": 2962.9687062524376
2736
+ },
2737
+ {
2738
+ "pair": "construction|yolo26x|Qwen3.8",
2739
+ "method": "confidence",
2740
+ "metric": 0.17813242189706796,
2741
+ "calls": 0.30459387483355527,
2742
+ "gflops": 29643.422601331185,
2743
+ "ms": 3038.174080777071
2744
+ },
2745
+ {
2746
+ "pair": "construction|yolo26x|Qwen3.8",
2747
+ "method": "confidence",
2748
+ "metric": 0.17772000842137498,
2749
+ "calls": 0.3129161118508655,
2750
+ "gflops": 30449.636920169738,
2751
+ "ms": 3119.919053086455
2752
+ },
2753
+ {
2754
+ "pair": "construction|yolo26x|Qwen3.8",
2755
+ "method": "confidence",
2756
+ "metric": 0.17791259734473341,
2757
+ "calls": 0.31990679094540614,
2758
+ "gflops": 31126.85694799413,
2759
+ "ms": 3188.5848298263377
2760
+ },
2761
+ {
2762
+ "pair": "construction|yolo26x|Qwen3.8",
2763
+ "method": "confidence",
2764
+ "metric": 0.1789653934599337,
2765
+ "calls": 0.3322237017310253,
2766
+ "gflops": 32320.05413987519,
2767
+ "ms": 3309.5673888442257
2768
+ },
2769
+ {
2770
+ "pair": "construction|yolo26x|Qwen3.8",
2771
+ "method": "confidence",
2772
+ "metric": 0.18042682200794885,
2773
+ "calls": 0.34054593874833555,
2774
+ "gflops": 33126.268458713756,
2775
+ "ms": 3391.31236115361
2776
+ },
2777
+ {
2778
+ "pair": "construction|yolo26x|Qwen3.8",
2779
+ "method": "confidence",
2780
+ "metric": 0.17962877325912951,
2781
+ "calls": 0.3488681757656458,
2782
+ "gflops": 33932.48277755231,
2783
+ "ms": 3473.057333462994
2784
+ },
2785
+ {
2786
+ "pair": "construction|yolo26x|Qwen3.8",
2787
+ "method": "confidence",
2788
+ "metric": 0.18158304685510065,
2789
+ "calls": 0.3581890812250333,
2790
+ "gflops": 34835.44281465149,
2791
+ "ms": 3564.611702449504
2792
+ },
2793
+ {
2794
+ "pair": "construction|yolo26x|Qwen3.8",
2795
+ "method": "confidence",
2796
+ "metric": 0.18069239723097547,
2797
+ "calls": 0.37017310252996005,
2798
+ "gflops": 35996.39143377901,
2799
+ "ms": 3682.3244625750167
2800
+ },
2801
+ {
2802
+ "pair": "construction|yolo26x|Qwen3.8",
2803
+ "method": "confidence",
2804
+ "metric": 0.1812845803851709,
2805
+ "calls": 0.37683089214380827,
2806
+ "gflops": 36641.362888849864,
2807
+ "ms": 3747.720440422524
2808
+ },
2809
+ {
2810
+ "pair": "construction|yolo26x|Qwen3.8",
2811
+ "method": "confidence",
2812
+ "metric": 0.1824781664689562,
2813
+ "calls": 0.38348868175765644,
2814
+ "gflops": 37286.3343439207,
2815
+ "ms": 3813.116418270031
2816
+ },
2817
+ {
2818
+ "pair": "construction|yolo26x|Qwen3.8",
2819
+ "method": "confidence",
2820
+ "metric": 0.18220594501399923,
2821
+ "calls": 0.4021304926764314,
2822
+ "gflops": 39092.254418119075,
2823
+ "ms": 3996.225156243051
2824
+ },
2825
+ {
2826
+ "pair": "construction|yolo26x|Qwen3.8",
2827
+ "method": "confidence",
2828
+ "metric": 0.18162498930772586,
2829
+ "calls": 0.41311584553928093,
2830
+ "gflops": 40156.457318985966,
2831
+ "ms": 4104.128519691438
2832
+ },
2833
+ {
2834
+ "pair": "construction|yolo26x|Qwen3.8",
2835
+ "method": "confidence",
2836
+ "metric": 0.18066434967152634,
2837
+ "calls": 0.42676431424766975,
2838
+ "gflops": 41478.6488018812,
2839
+ "ms": 4238.190274278827
2840
+ },
2841
+ {
2842
+ "pair": "construction|yolo26x|Qwen3.8",
2843
+ "method": "confidence",
2844
+ "metric": 0.1803913789289444,
2845
+ "calls": 0.4324234354194407,
2846
+ "gflops": 42026.874538691416,
2847
+ "ms": 4293.776855449209
2848
+ },
2849
+ {
2850
+ "pair": "construction|yolo26x|Qwen3.8",
2851
+ "method": "confidence",
2852
+ "metric": 0.17959468902865128,
2853
+ "calls": 0.44107856191744343,
2854
+ "gflops": 42865.33743028352,
2855
+ "ms": 4378.791626650968
2856
+ },
2857
+ {
2858
+ "pair": "construction|yolo26x|Qwen3.8",
2859
+ "method": "confidence",
2860
+ "metric": 0.17863383514615866,
2861
+ "calls": 0.45772303595206393,
2862
+ "gflops": 44477.766067960634,
2863
+ "ms": 4542.281571269737
2864
+ },
2865
+ {
2866
+ "pair": "construction|yolo26x|Qwen3.8",
2867
+ "method": "confidence",
2868
+ "metric": 0.17957308538379646,
2869
+ "calls": 0.4717043941411451,
2870
+ "gflops": 45832.20612360941,
2871
+ "ms": 4679.6131247495005
2872
+ },
2873
+ {
2874
+ "pair": "construction|yolo26x|Qwen3.8",
2875
+ "method": "confidence",
2876
+ "metric": 0.17910245145856155,
2877
+ "calls": 0.4826897470039947,
2878
+ "gflops": 46896.40902447631,
2879
+ "ms": 4787.516488197888
2880
+ },
2881
+ {
2882
+ "pair": "construction|yolo26x|Qwen3.8",
2883
+ "method": "confidence",
2884
+ "metric": 0.17871833907402393,
2885
+ "calls": 0.4896804260985353,
2886
+ "gflops": 47573.62905230069,
2887
+ "ms": 4856.182264937771
2888
+ },
2889
+ {
2890
+ "pair": "construction|yolo26x|Qwen3.8",
2891
+ "method": "confidence",
2892
+ "metric": 0.18015785955296285,
2893
+ "calls": 0.5013315579227696,
2894
+ "gflops": 48702.32909867467,
2895
+ "ms": 4970.625226170908
2896
+ },
2897
+ {
2898
+ "pair": "construction|yolo26x|Qwen3.8",
2899
+ "method": "confidence",
2900
+ "metric": 0.1810878167493375,
2901
+ "calls": 0.5116511318242344,
2902
+ "gflops": 49702.034854034486,
2903
+ "ms": 5071.988991834544
2904
+ },
2905
+ {
2906
+ "pair": "construction|yolo26x|Qwen3.8",
2907
+ "method": "confidence",
2908
+ "metric": 0.18177962418520882,
2909
+ "calls": 0.5183089214380826,
2910
+ "gflops": 50347.00630910533,
2911
+ "ms": 5137.384969682052
2912
+ },
2913
+ {
2914
+ "pair": "construction|yolo26x|Qwen3.8",
2915
+ "method": "confidence",
2916
+ "metric": 0.18246323539519374,
2917
+ "calls": 0.5252996005326231,
2918
+ "gflops": 51024.226336929714,
2919
+ "ms": 5206.050746421934
2920
+ },
2921
+ {
2922
+ "pair": "construction|yolo26x|Qwen3.8",
2923
+ "method": "confidence",
2924
+ "metric": 0.1839876166906015,
2925
+ "calls": 0.5396138482023968,
2926
+ "gflops": 52410.914965332035,
2927
+ "ms": 5346.652098794074
2928
+ },
2929
+ {
2930
+ "pair": "construction|yolo26x|Qwen3.8",
2931
+ "method": "confidence",
2932
+ "metric": 0.1846674213907964,
2933
+ "calls": 0.5452729693741678,
2934
+ "gflops": 52959.14070214226,
2935
+ "ms": 5402.2386799644555
2936
+ },
2937
+ {
2938
+ "pair": "construction|yolo26x|Qwen3.8",
2939
+ "method": "confidence",
2940
+ "metric": 0.18676294233791663,
2941
+ "calls": 0.559254327563249,
2942
+ "gflops": 54313.58075779103,
2943
+ "ms": 5539.57023344422
2944
+ },
2945
+ {
2946
+ "pair": "construction|yolo26x|Qwen3.8",
2947
+ "method": "confidence",
2948
+ "metric": 0.18938663221175195,
2949
+ "calls": 0.572237017310253,
2950
+ "gflops": 55571.275095179175,
2951
+ "ms": 5667.092390246859
2952
+ },
2953
+ {
2954
+ "pair": "construction|yolo26x|Qwen3.8",
2955
+ "method": "confidence",
2956
+ "metric": 0.1909571002676625,
2957
+ "calls": 0.5862183754993342,
2958
+ "gflops": 56925.71515082795,
2959
+ "ms": 5804.423943726624
2960
+ },
2961
+ {
2962
+ "pair": "construction|yolo26x|Qwen3.8",
2963
+ "method": "confidence",
2964
+ "metric": 0.1934300101790686,
2965
+ "calls": 0.5998668442077231,
2966
+ "gflops": 58247.90663372319,
2967
+ "ms": 5938.485698314014
2968
+ },
2969
+ {
2970
+ "pair": "construction|yolo26x|Qwen3.8",
2971
+ "method": "confidence",
2972
+ "metric": 0.1973701682130997,
2973
+ "calls": 0.6191744340878829,
2974
+ "gflops": 60118.32385342864,
2975
+ "ms": 6128.134034071785
2976
+ },
2977
+ {
2978
+ "pair": "construction|yolo26x|Qwen3.8",
2979
+ "method": "confidence",
2980
+ "metric": 0.1986727924998457,
2981
+ "calls": 0.6264980026631158,
2982
+ "gflops": 60827.792454006565,
2983
+ "ms": 6200.069609704043
2984
+ },
2985
+ {
2986
+ "pair": "construction|yolo26x|Qwen3.8",
2987
+ "method": "confidence",
2988
+ "metric": 0.20186289191698192,
2989
+ "calls": 0.6384820239680427,
2990
+ "gflops": 61988.74107313409,
2991
+ "ms": 6317.782369829556
2992
+ },
2993
+ {
2994
+ "pair": "construction|yolo26x|Qwen3.8",
2995
+ "method": "confidence",
2996
+ "metric": 0.20246031058520178,
2997
+ "calls": 0.6461384820239681,
2998
+ "gflops": 62730.45824646557,
2999
+ "ms": 6392.987744354189
3000
+ },
3001
+ {
3002
+ "pair": "construction|yolo26x|Qwen3.8",
3003
+ "method": "confidence",
3004
+ "metric": 0.20349944137622245,
3005
+ "calls": 0.653462050599201,
3006
+ "gflops": 63439.92684704349,
3007
+ "ms": 6464.923319986447
3008
+ },
3009
+ {
3010
+ "pair": "construction|yolo26x|Qwen3.8",
3011
+ "method": "confidence",
3012
+ "metric": 0.20442990090722246,
3013
+ "calls": 0.6597869507323568,
3014
+ "gflops": 64052.64972936079,
3015
+ "ms": 6527.049498941578
3016
+ },
3017
+ {
3018
+ "pair": "construction|yolo26x|Qwen3.8",
3019
+ "method": "confidence",
3020
+ "metric": 0.2044993942500819,
3021
+ "calls": 0.6661118508655126,
3022
+ "gflops": 64665.372611678096,
3023
+ "ms": 6589.17567789671
3024
+ },
3025
+ {
3026
+ "pair": "construction|yolo26x|Qwen3.8",
3027
+ "method": "confidence",
3028
+ "metric": 0.2064379542733177,
3029
+ "calls": 0.6734354194407457,
3030
+ "gflops": 65374.84121225603,
3031
+ "ms": 6661.1112535289685
3032
+ },
3033
+ {
3034
+ "pair": "construction|yolo26x|Qwen3.8",
3035
+ "method": "confidence",
3036
+ "metric": 0.2065831206227621,
3037
+ "calls": 0.6820905459387483,
3038
+ "gflops": 66213.30410384812,
3039
+ "ms": 6746.126024730727
3040
+ },
3041
+ {
3042
+ "pair": "construction|yolo26x|Qwen3.8",
3043
+ "method": "confidence",
3044
+ "metric": 0.20802802394105854,
3045
+ "calls": 0.6927430093209055,
3046
+ "gflops": 67245.25843196148,
3047
+ "ms": 6850.759589286739
3048
+ },
3049
+ {
3050
+ "pair": "construction|yolo26x|Qwen3.8",
3051
+ "method": "confidence",
3052
+ "metric": 0.21169680596278434,
3053
+ "calls": 0.7090545938748336,
3054
+ "gflops": 68825.43849688505,
3055
+ "ms": 7010.979735013132
3056
+ },
3057
+ {
3058
+ "pair": "construction|yolo26x|Qwen3.8",
3059
+ "method": "confidence",
3060
+ "metric": 0.2139280527103987,
3061
+ "calls": 0.7170439414114514,
3062
+ "gflops": 69599.40424297006,
3063
+ "ms": 7089.454908430141
3064
+ },
3065
+ {
3066
+ "pair": "construction|yolo26x|Qwen3.8",
3067
+ "method": "confidence",
3068
+ "metric": 0.2146900231025185,
3069
+ "calls": 0.7243675099866844,
3070
+ "gflops": 70308.87284354799,
3071
+ "ms": 7161.390484062398
3072
+ },
3073
+ {
3074
+ "pair": "construction|yolo26x|Qwen3.8",
3075
+ "method": "confidence",
3076
+ "metric": 0.21653561577692484,
3077
+ "calls": 0.7330226364846871,
3078
+ "gflops": 71147.33573514009,
3079
+ "ms": 7246.405255264158
3080
+ },
3081
+ {
3082
+ "pair": "construction|yolo26x|Qwen3.8",
3083
+ "method": "confidence",
3084
+ "metric": 0.21721689648133474,
3085
+ "calls": 0.7363515312916112,
3086
+ "gflops": 71469.82146267552,
3087
+ "ms": 7279.1032441879115
3088
+ },
3089
+ {
3090
+ "pair": "construction|yolo26x|Qwen3.8",
3091
+ "method": "confidence",
3092
+ "metric": 0.22103445955201576,
3093
+ "calls": 0.7486684420772304,
3094
+ "gflops": 72663.01865455658,
3095
+ "ms": 7400.0858032057995
3096
+ },
3097
+ {
3098
+ "pair": "construction|yolo26x|Qwen3.8",
3099
+ "method": "confidence",
3100
+ "metric": 0.22340134248367294,
3101
+ "calls": 0.757323568575233,
3102
+ "gflops": 73501.48154614867,
3103
+ "ms": 7485.100574407558
3104
+ },
3105
+ {
3106
+ "pair": "construction|yolo26x|Qwen3.8",
3107
+ "method": "confidence",
3108
+ "metric": 0.2269322568854726,
3109
+ "calls": 0.7639813581890812,
3110
+ "gflops": 74146.45300121952,
3111
+ "ms": 7550.496552255066
3112
+ },
3113
+ {
3114
+ "pair": "construction|yolo26x|Qwen3.8",
3115
+ "method": "confidence",
3116
+ "metric": 0.22757452798038888,
3117
+ "calls": 0.7733022636484687,
3118
+ "gflops": 75049.4130383187,
3119
+ "ms": 7642.050921241575
3120
+ },
3121
+ {
3122
+ "pair": "construction|yolo26x|Qwen3.8",
3123
+ "method": "confidence",
3124
+ "metric": 0.227821050912831,
3125
+ "calls": 0.784953395472703,
3126
+ "gflops": 76178.11308469268,
3127
+ "ms": 7756.493882474713
3128
+ },
3129
+ {
3130
+ "pair": "construction|yolo26x|Qwen3.8",
3131
+ "method": "confidence",
3132
+ "metric": 0.22991865671924006,
3133
+ "calls": 0.7939414114513982,
3134
+ "gflops": 77048.82454903833,
3135
+ "ms": 7844.7784525688485
3136
+ },
3137
+ {
3138
+ "pair": "construction|yolo26x|Qwen3.8",
3139
+ "method": "confidence",
3140
+ "metric": 0.23486191981046528,
3141
+ "calls": 0.8039280958721704,
3142
+ "gflops": 78016.28173164459,
3143
+ "ms": 7942.8724193401085
3144
+ },
3145
+ {
3146
+ "pair": "construction|yolo26x|Qwen3.8",
3147
+ "method": "confidence",
3148
+ "metric": 0.23803916047544685,
3149
+ "calls": 0.8122503328894807,
3150
+ "gflops": 78822.49605048315,
3151
+ "ms": 8024.617391649493
3152
+ },
3153
+ {
3154
+ "pair": "construction|yolo26x|Qwen3.8",
3155
+ "method": "confidence",
3156
+ "metric": 0.24022947987632814,
3157
+ "calls": 0.8215712383488681,
3158
+ "gflops": 79725.45608758234,
3159
+ "ms": 8116.171760636003
3160
+ },
3161
+ {
3162
+ "pair": "construction|yolo26x|Qwen3.8",
3163
+ "method": "confidence",
3164
+ "metric": 0.24528627405740036,
3165
+ "calls": 0.8388814913448736,
3166
+ "gflops": 81402.38187076653,
3167
+ "ms": 8286.201303039523
3168
+ },
3169
+ {
3170
+ "pair": "construction|yolo26x|Qwen3.8",
3171
+ "method": "confidence",
3172
+ "metric": 0.249537136326177,
3173
+ "calls": 0.8535286284953395,
3174
+ "gflops": 82821.3190719224,
3175
+ "ms": 8430.072454304036
3176
+ },
3177
+ {
3178
+ "pair": "construction|yolo26x|Qwen3.8",
3179
+ "method": "confidence",
3180
+ "metric": 0.25211074301800224,
3181
+ "calls": 0.8638482023968043,
3182
+ "gflops": 83821.0248272822,
3183
+ "ms": 8531.436219967674
3184
+ },
3185
+ {
3186
+ "pair": "construction|yolo26x|Qwen3.8",
3187
+ "method": "confidence",
3188
+ "metric": 0.2540042317315216,
3189
+ "calls": 0.8745006657789614,
3190
+ "gflops": 84852.97915539556,
3191
+ "ms": 8636.069784523685
3192
+ },
3193
+ {
3194
+ "pair": "construction|yolo26x|Qwen3.8",
3195
+ "method": "confidence",
3196
+ "metric": 0.2540996517958595,
3197
+ "calls": 0.8764980026631158,
3198
+ "gflops": 85046.4705919168,
3199
+ "ms": 8655.688577877936
3200
+ },
3201
+ {
3202
+ "pair": "construction|yolo26x|Qwen3.8",
3203
+ "method": "confidence",
3204
+ "metric": 0.25436194678023316,
3205
+ "calls": 0.8841544607190412,
3206
+ "gflops": 85788.18776524828,
3207
+ "ms": 8730.89395240257
3208
+ },
3209
+ {
3210
+ "pair": "construction|yolo26x|Qwen3.8",
3211
+ "method": "confidence",
3212
+ "metric": 0.2534339447803836,
3213
+ "calls": 0.8891478029294274,
3214
+ "gflops": 86271.91635655142,
3215
+ "ms": 8779.9409357882
3216
+ },
3217
+ {
3218
+ "pair": "construction|yolo26x|Qwen3.8",
3219
+ "method": "confidence",
3220
+ "metric": 0.2563240880787248,
3221
+ "calls": 0.8981358189081226,
3222
+ "gflops": 87142.62782089706,
3223
+ "ms": 8868.225505882336
3224
+ },
3225
+ {
3226
+ "pair": "construction|yolo26x|Qwen3.8",
3227
+ "method": "confidence",
3228
+ "metric": 0.25551845731687656,
3229
+ "calls": 0.9054593874833555,
3230
+ "gflops": 87852.09642147498,
3231
+ "ms": 8940.161081514594
3232
+ },
3233
+ {
3234
+ "pair": "construction|yolo26x|Qwen3.8",
3235
+ "method": "confidence",
3236
+ "metric": 0.2597490004762105,
3237
+ "calls": 0.9221038615179761,
3238
+ "gflops": 89464.5250591521,
3239
+ "ms": 9103.651026133362
3240
+ },
3241
+ {
3242
+ "pair": "construction|yolo26x|Qwen3.8",
3243
+ "method": "confidence",
3244
+ "metric": 0.26306323674307786,
3245
+ "calls": 0.9254327563249002,
3246
+ "gflops": 89787.01078668753,
3247
+ "ms": 9136.349015057116
3248
+ },
3249
+ {
3250
+ "pair": "construction|yolo26x|Qwen3.8",
3251
+ "method": "confidence",
3252
+ "metric": 0.26218185491449153,
3253
+ "calls": 0.9364181091877497,
3254
+ "gflops": 90851.21368755442,
3255
+ "ms": 9244.252378505502
3256
+ },
3257
+ {
3258
+ "pair": "construction|yolo26x|Qwen3.8",
3259
+ "method": "confidence",
3260
+ "metric": 0.26357052137555514,
3261
+ "calls": 0.953728362183755,
3262
+ "gflops": 92528.13947073862,
3263
+ "ms": 9414.281920909021
3264
+ },
3265
+ {
3266
+ "pair": "construction|yolo26x|Qwen3.8",
3267
+ "method": "confidence",
3268
+ "metric": 0.26400290307863034,
3269
+ "calls": 0.96271637816245,
3270
+ "gflops": 93398.85093508427,
3271
+ "ms": 9502.566491003154
3272
+ },
3273
+ {
3274
+ "pair": "construction|yolo26x|Qwen3.8",
3275
+ "method": "confidence",
3276
+ "metric": 0.2647475724296899,
3277
+ "calls": 0.9700399467376831,
3278
+ "gflops": 94108.3195356622,
3279
+ "ms": 9574.502066635414
3280
+ },
3281
+ {
3282
+ "pair": "construction|yolo26x|Qwen3.8",
3283
+ "method": "confidence",
3284
+ "metric": 0.2649306125590882,
3285
+ "calls": 0.987017310252996,
3286
+ "gflops": 95752.99674609285,
3287
+ "ms": 9741.261810146556
3288
+ },
3289
+ {
3290
+ "pair": "construction|yolo26x|Qwen3.8",
3291
+ "method": "confidence",
3292
+ "metric": 0.27001009067886245,
3293
+ "calls": 1.0,
3294
+ "gflops": 97010.691083481,
3295
+ "ms": 9868.783966949195
3296
+ }
3297
+ ],
3298
+ "protocol": "Five-seed mean call rates times existing measured mean component costs; not integrated latency measurements."
3299
+ }
releases/20260913_lr/paper_data/earlier_results.json ADDED
The diff for this file is too large to render. See raw diff
 
releases/20260913_lr/paper_data/fig2_audit/remote_counts.json ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "expert": "yolo26x",
4
+ "endpoint": "MiniCPM",
5
+ "images": 3004,
6
+ "counts": {
7
+ "both_correct": 8176,
8
+ "negative": 11580,
9
+ "both_correct_negative": 7940,
10
+ "rescuable": 1198,
11
+ "harmful": 2109,
12
+ "positive": 436,
13
+ "both_wrong": 533,
14
+ "both_correct_positive": 236,
15
+ "null_predictions": 112,
16
+ "metric_vs_state_difference": 103,
17
+ "difference_null": 103
18
+ }
19
+ },
20
+ {
21
+ "expert": "yolo26x",
22
+ "endpoint": "Qwen3.8",
23
+ "images": 3004,
24
+ "counts": {
25
+ "both_correct": 9508,
26
+ "negative": 11580,
27
+ "both_correct_negative": 9198,
28
+ "rescuable": 1282,
29
+ "positive": 436,
30
+ "both_correct_positive": 310,
31
+ "both_wrong": 449,
32
+ "harmful": 777,
33
+ "null_predictions": 36,
34
+ "metric_vs_state_difference": 34,
35
+ "difference_null": 34
36
+ }
37
+ },
38
+ {
39
+ "expert": "rtdetr_x_fp32",
40
+ "endpoint": "MiniCPM",
41
+ "images": 3004,
42
+ "counts": {
43
+ "rescuable": 3314,
44
+ "negative": 11580,
45
+ "both_correct": 6060,
46
+ "both_correct_negative": 5846,
47
+ "harmful": 1945,
48
+ "positive": 436,
49
+ "both_wrong": 697,
50
+ "both_correct_positive": 214,
51
+ "null_predictions": 112,
52
+ "metric_vs_state_difference": 103,
53
+ "difference_null": 103
54
+ }
55
+ },
56
+ {
57
+ "expert": "rtdetr_x_fp32",
58
+ "endpoint": "Qwen3.8",
59
+ "images": 3004,
60
+ "counts": {
61
+ "rescuable": 3461,
62
+ "negative": 11580,
63
+ "both_correct": 7329,
64
+ "both_correct_negative": 7036,
65
+ "positive": 436,
66
+ "both_correct_positive": 293,
67
+ "both_wrong": 550,
68
+ "harmful": 676,
69
+ "null_predictions": 36,
70
+ "metric_vs_state_difference": 34,
71
+ "difference_null": 34
72
+ }
73
+ }
74
+ ]
releases/20260913_lr/paper_data/fig2_audit/remote_counts.py ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from pathlib import Path
2
+ import json
3
+ from collections import Counter
4
+
5
+ base=Path('/mnt/HithinkOmniSSD/user_workspace/caisihang/research/ICASSP_ModelCollaboration/outputs/router_full_20260910/data')
6
+ out=[]
7
+ for expert in ['yolo26x','rtdetr_x_fp32']:
8
+ rows=[json.loads(s) for s in (base/expert/'test/records.jsonl').read_text().splitlines()]
9
+ for endpoint in ['MiniCPM','Qwen3.8']:
10
+ c=Counter()
11
+ for r in rows:
12
+ for event,s in r['small_label']['events'].items():
13
+ l=r['large_labels'][endpoint]['events'][event]
14
+ assert s['target']==l['target']
15
+ state={(True,True):'both_correct',(False,True):'rescuable',(True,False):'harmful',(False,False):'both_wrong'}[(bool(s['correct']),bool(l['correct']))]
16
+ c[state]+=1
17
+ c['positive' if s['target'] else 'negative']+=1
18
+ if state=='both_correct':c['both_correct_positive' if s['target'] else 'both_correct_negative']+=1
19
+ if l['prediction'] is None:c['null_predictions']+=1
20
+ score_correct=(l['prediction'] is True)==bool(l['target'])
21
+ if score_correct!=bool(l['correct']):
22
+ c['metric_vs_state_difference']+=1
23
+ c['difference_null' if l['prediction'] is None else 'difference_other']+=1
24
+ out.append(dict(expert=expert,endpoint=endpoint,images=len(rows),counts=dict(c)))
25
+ print(json.dumps(out,indent=2))
releases/20260913_lr/paper_data/five_seed_counts.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "better_metric": 11,
3
+ "fewer_calls": 9,
4
+ "both": 8,
5
+ "above_endpoints": 10,
6
+ "better_ranked_area": 16
7
+ }
releases/20260913_lr/paper_data/full_budget_results.json ADDED
The diff for this file is too large to render. See raw diff
 
releases/20260913_lr/paper_data/historical_results_v44.json ADDED
The diff for this file is too large to render. See raw diff
 
releases/20260913_lr/paper_data/joint_MiniCPM_comparison_20260910.json ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "expert": "cub",
4
+ "endpoint": "MiniCPM",
5
+ "single_task_test_fixed25": 0.7119433897134967,
6
+ "multi_task_test_fixed25": 0.7079737659647911,
7
+ "delta_multi_minus_single": -0.003969623748705575
8
+ },
9
+ {
10
+ "expert": "glsim",
11
+ "endpoint": "MiniCPM",
12
+ "single_task_test_fixed25": 0.759233690024163,
13
+ "multi_task_test_fixed25": 0.7745944080082844,
14
+ "delta_multi_minus_single": 0.01536071798412142
15
+ },
16
+ {
17
+ "expert": "nlvr2",
18
+ "endpoint": "MiniCPM",
19
+ "single_task_test_fixed25": 0.8346846523642104,
20
+ "multi_task_test_fixed25": 0.8309535768099304,
21
+ "delta_multi_minus_single": -0.0037310755542799834
22
+ },
23
+ {
24
+ "expert": "vilt",
25
+ "endpoint": "MiniCPM",
26
+ "single_task_test_fixed25": 0.8046925450240368,
27
+ "multi_task_test_fixed25": 0.802611752887996,
28
+ "delta_multi_minus_single": -0.002080792136040732
29
+ },
30
+ {
31
+ "expert": "groundingdino",
32
+ "endpoint": "MiniCPM",
33
+ "single_task_test_fixed25": 0.3598105663159686,
34
+ "multi_task_test_fixed25": 0.37004792558772653,
35
+ "delta_multi_minus_single": 0.010237359271757951
36
+ },
37
+ {
38
+ "expert": "instancevg",
39
+ "endpoint": "MiniCPM",
40
+ "single_task_test_fixed25": 0.6394237586138446,
41
+ "multi_task_test_fixed25": 0.6243654822335025,
42
+ "delta_multi_minus_single": -0.015058276380342073
43
+ },
44
+ {
45
+ "expert": "yolo26x",
46
+ "endpoint": "MiniCPM",
47
+ "single_task_test_fixed25": 0.18032734231886507,
48
+ "multi_task_test_fixed25": 0.1836164473038217,
49
+ "delta_multi_minus_single": 0.003289104984956631
50
+ },
51
+ {
52
+ "expert": "rtdetr_x_fp32",
53
+ "endpoint": "MiniCPM",
54
+ "single_task_test_fixed25": 0.10367040308754236,
55
+ "multi_task_test_fixed25": 0.11260360812088349,
56
+ "delta_multi_minus_single": 0.008933205033341135
57
+ }
58
+ ]
releases/20260913_lr/paper_data/joint_Qwen3.8_comparison_20260910.json ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "expert": "cub",
4
+ "endpoint": "Qwen3.8",
5
+ "single_task_test_fixed25": 0.7619951674145667,
6
+ "multi_task_test_fixed25": 0.7782188470831896,
7
+ "delta_multi_minus_single": 0.016223679668622815
8
+ },
9
+ {
10
+ "expert": "glsim",
11
+ "endpoint": "Qwen3.8",
12
+ "single_task_test_fixed25": 0.8301691404901622,
13
+ "multi_task_test_fixed25": 0.8448394891266828,
14
+ "delta_multi_minus_single": 0.014670348636520614
15
+ },
16
+ {
17
+ "expert": "nlvr2",
18
+ "endpoint": "Qwen3.8",
19
+ "single_task_test_fixed25": 0.8372677046710196,
20
+ "multi_task_test_fixed25": 0.8337518834756403,
21
+ "delta_multi_minus_single": -0.0035158211953792984
22
+ },
23
+ {
24
+ "expert": "vilt",
25
+ "endpoint": "Qwen3.8",
26
+ "single_task_test_fixed25": 0.8045490421181029,
27
+ "multi_task_test_fixed25": 0.8067733371600775,
28
+ "delta_multi_minus_single": 0.0022242950419746332
29
+ },
30
+ {
31
+ "expert": "groundingdino",
32
+ "endpoint": "Qwen3.8",
33
+ "single_task_test_fixed25": 0.4223974137197629,
34
+ "multi_task_test_fixed25": 0.45991549215892014,
35
+ "delta_multi_minus_single": 0.03751807843915722
36
+ },
37
+ {
38
+ "expert": "instancevg",
39
+ "endpoint": "Qwen3.8",
40
+ "single_task_test_fixed25": 0.7224853245611548,
41
+ "multi_task_test_fixed25": 0.7053001729858492,
42
+ "delta_multi_minus_single": -0.017185151575305535
43
+ },
44
+ {
45
+ "expert": "yolo26x",
46
+ "endpoint": "Qwen3.8",
47
+ "single_task_test_fixed25": 0.19009376433100783,
48
+ "multi_task_test_fixed25": 0.19603467084956394,
49
+ "delta_multi_minus_single": 0.0059409065185561105
50
+ },
51
+ {
52
+ "expert": "rtdetr_x_fp32",
53
+ "endpoint": "Qwen3.8",
54
+ "single_task_test_fixed25": 0.10483176081963008,
55
+ "multi_task_test_fixed25": 0.12038495915476813,
56
+ "delta_multi_minus_single": 0.015553198335138047
57
+ }
58
+ ]
releases/20260913_lr/paper_data/layout_verification.json ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "old_distribution": {
3
+ "width_pt": 523.1995239257812,
4
+ "height_pt": 309.7997741699219,
5
+ "height_at_7_16_in": 4.239618504261674
6
+ },
7
+ "old_budget": {
8
+ "width_pt": 526.7994995117188,
9
+ "height_pt": 237.7997589111328,
10
+ "height_at_7_16_in": 3.2320575007794505
11
+ },
12
+ "combined": {
13
+ "width_pt": 515.52001953125,
14
+ "height_pt": 324.0,
15
+ "height_at_7_16_in": 4.499999829510743
16
+ },
17
+ "reduction_percent": 39.77255134624947,
18
+ "old_pages": 16,
19
+ "new_pages": 14,
20
+ "new_figure_and_section_pages": [
21
+ 6,
22
+ 7
23
+ ],
24
+ "changed_sections": [
25
+ "sections\\appendix.tex",
26
+ "sections\\experiments.tex"
27
+ ],
28
+ "figure1_identical": true
29
+ }
releases/20260913_lr/paper_data/lr_control_curves.json ADDED
The diff for this file is too large to render. See raw diff
 
releases/20260913_lr/paper_data/lr_control_summary.json ADDED
@@ -0,0 +1,156 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "4S_difference": {
3
+ "cub": {
4
+ "full": 71.30477624717177,
5
+ "middle_high": 66.65309176574908
6
+ },
7
+ "grefcoco": {
8
+ "full": 60.10334108982569,
9
+ "middle_high": 62.55288392181684
10
+ },
11
+ "nlvr2": {
12
+ "full": 86.42688661827373,
13
+ "middle_high": 87.82059502279398
14
+ },
15
+ "construction": {
16
+ "full": 21.68664457361428,
17
+ "middle_high": 23.37179473046792
18
+ },
19
+ "overall": {
20
+ "mean": 59.880412132221366,
21
+ "std": 0.06230299379475004
22
+ }
23
+ },
24
+ "4S_RH": {
25
+ "cub": {
26
+ "full": 67.80147679642886,
27
+ "middle_high": 62.716073314651304
28
+ },
29
+ "grefcoco": {
30
+ "full": 59.83696029450736,
31
+ "middle_high": 62.2898964392558
32
+ },
33
+ "nlvr2": {
34
+ "full": 87.36051668387572,
35
+ "middle_high": 89.08451490248436
36
+ },
37
+ "construction": {
38
+ "full": 21.11319259968111,
39
+ "middle_high": 22.83140793334926
40
+ },
41
+ "overall": {
42
+ "mean": 59.028036593623256,
43
+ "std": 0.09215446832595518
44
+ }
45
+ },
46
+ "4S_error_ratio": {
47
+ "cub": {
48
+ "full": 67.63433680158339,
49
+ "middle_high": 62.495205558823486
50
+ },
51
+ "grefcoco": {
52
+ "full": 59.9030469001769,
53
+ "middle_high": 62.255919404292584
54
+ },
55
+ "nlvr2": {
56
+ "full": 87.36099315635701,
57
+ "middle_high": 89.08882695682252
58
+ },
59
+ "construction": {
60
+ "full": 21.29756392944788,
61
+ "middle_high": 22.972511633206114
62
+ },
63
+ "overall": {
64
+ "mean": 59.048985196891294,
65
+ "std": 0.10043056570886838
66
+ }
67
+ },
68
+ "4S_normalized": {
69
+ "cub": {
70
+ "full": 71.3715854777698,
71
+ "middle_high": 66.74755638479657
72
+ },
73
+ "grefcoco": {
74
+ "full": 60.65894058616111,
75
+ "middle_high": 63.22949855130163
76
+ },
77
+ "nlvr2": {
78
+ "full": 87.36085314912711,
79
+ "middle_high": 89.0886402849806
80
+ },
81
+ "construction": {
82
+ "full": 21.625047131135076,
83
+ "middle_high": 23.29240515579527
84
+ },
85
+ "overall": {
86
+ "mean": 60.25410658604827,
87
+ "std": 0.055784402767809056
88
+ }
89
+ },
90
+ "three": {
91
+ "cub": {
92
+ "full": 71.27395612317933,
93
+ "middle_high": 66.60007988650366
94
+ },
95
+ "grefcoco": {
96
+ "full": 60.12755657790998,
97
+ "middle_high": 62.59349605468297
98
+ },
99
+ "nlvr2": {
100
+ "full": 86.340413568538,
101
+ "middle_high": 87.71263160064547
102
+ },
103
+ "construction": {
104
+ "full": 21.76578337259674,
105
+ "middle_high": 23.45434250634451
106
+ },
107
+ "overall": {
108
+ "mean": 59.876927410556014,
109
+ "std": 0.06459211304335916
110
+ }
111
+ },
112
+ "gain": {
113
+ "cub": {
114
+ "full": 71.01409198260043,
115
+ "middle_high": 66.28415678198927
116
+ },
117
+ "grefcoco": {
118
+ "full": 59.96696456319969,
119
+ "middle_high": 62.41541160228904
120
+ },
121
+ "nlvr2": {
122
+ "full": 86.29461067562258,
123
+ "middle_high": 87.67252770031322
124
+ },
125
+ "construction": {
126
+ "full": 21.60783215955454,
127
+ "middle_high": 23.264626260993804
128
+ },
129
+ "overall": {
130
+ "mean": 59.72087484524432,
131
+ "std": 0.039237511453832113
132
+ }
133
+ },
134
+ "posthoc": {
135
+ "cub": {
136
+ "full": 65.85980220810953,
137
+ "middle_high": 60.37970343902742
138
+ },
139
+ "grefcoco": {
140
+ "full": 59.367755773800184,
141
+ "middle_high": 61.58463842152389
142
+ },
143
+ "nlvr2": {
144
+ "full": 87.32516457516526,
145
+ "middle_high": 89.03494498253306
146
+ },
147
+ "construction": {
148
+ "full": 21.23728513819176,
149
+ "middle_high": 22.948980499229855
150
+ },
151
+ "overall": {
152
+ "mean": 58.44750192381669,
153
+ "std": 0.0661360515853849
154
+ }
155
+ }
156
+ }
releases/20260913_lr/paper_data/lr_main_summary.json ADDED
@@ -0,0 +1,210 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "key": [
4
+ "cub",
5
+ "cub",
6
+ "MiniCPM"
7
+ ],
8
+ "confidence_score": 81.429064549534,
9
+ "confidence_calls": 0.0,
10
+ "lr_score": 81.42906454953402,
11
+ "lr_calls": 0.5937176389368313,
12
+ "rescued": 4.6,
13
+ "harmed": 4.6
14
+ },
15
+ {
16
+ "key": [
17
+ "cub",
18
+ "cub",
19
+ "Qwen3.8"
20
+ ],
21
+ "confidence_score": 81.429064549534,
22
+ "confidence_calls": 0.0,
23
+ "lr_score": 81.42906454953402,
24
+ "lr_calls": 0.0,
25
+ "rescued": 0.0,
26
+ "harmed": 0.0
27
+ },
28
+ {
29
+ "key": [
30
+ "cub",
31
+ "glsim",
32
+ "MiniCPM"
33
+ ],
34
+ "confidence_score": 90.16223679668623,
35
+ "confidence_calls": 0.0,
36
+ "lr_score": 90.13117017604418,
37
+ "lr_calls": 0.8491542975491888,
38
+ "rescued": 1.0,
39
+ "harmed": 2.8
40
+ },
41
+ {
42
+ "key": [
43
+ "cub",
44
+ "glsim",
45
+ "Qwen3.8"
46
+ ],
47
+ "confidence_score": 90.16223679668623,
48
+ "confidence_calls": 0.0,
49
+ "lr_score": 90.16223679668623,
50
+ "lr_calls": 0.0,
51
+ "rescued": 0.0,
52
+ "harmed": 0.0
53
+ },
54
+ {
55
+ "key": [
56
+ "grefcoco",
57
+ "groundingdino",
58
+ "MiniCPM"
59
+ ],
60
+ "confidence_score": 50.00708958398321,
61
+ "confidence_calls": 100.0,
62
+ "lr_score": 50.22828460425942,
63
+ "lr_calls": 58.18676800045374,
64
+ "rescued": 9475.4,
65
+ "harmed": 1174.4
66
+ },
67
+ {
68
+ "key": [
69
+ "grefcoco",
70
+ "groundingdino",
71
+ "Qwen3.8"
72
+ ],
73
+ "confidence_score": 74.47182599325072,
74
+ "confidence_calls": 100.0,
75
+ "lr_score": 75.12179905283158,
76
+ "lr_calls": 88.46666477611093,
77
+ "rescued": 18720.8,
78
+ "harmed": 1641.6
79
+ },
80
+ {
81
+ "key": [
82
+ "grefcoco",
83
+ "instancevg",
84
+ "MiniCPM"
85
+ ],
86
+ "confidence_score": 61.877889005473165,
87
+ "confidence_calls": 52.09426310864078,
88
+ "lr_score": 67.91367722542041,
89
+ "lr_calls": 19.038085245157816,
90
+ "rescued": 727.6,
91
+ "harmed": 388.2
92
+ },
93
+ {
94
+ "key": [
95
+ "grefcoco",
96
+ "instancevg",
97
+ "Qwen3.8"
98
+ ],
99
+ "confidence_score": 72.43286163967898,
100
+ "confidence_calls": 68.38612710206165,
101
+ "lr_score": 76.24365482233502,
102
+ "lr_calls": 62.00153135014037,
103
+ "rescued": 4753.6,
104
+ "harmed": 1476.8
105
+ },
106
+ {
107
+ "key": [
108
+ "nlvr2",
109
+ "nlvr2",
110
+ "MiniCPM"
111
+ ],
112
+ "confidence_score": 90.77276314845376,
113
+ "confidence_calls": 85.80038745784601,
114
+ "lr_score": 90.80002870058121,
115
+ "lr_calls": 91.10425486116092,
116
+ "rescued": 2189.4,
117
+ "harmed": 684.6
118
+ },
119
+ {
120
+ "key": [
121
+ "nlvr2",
122
+ "nlvr2",
123
+ "Qwen3.8"
124
+ ],
125
+ "confidence_score": 90.24180239649853,
126
+ "confidence_calls": 85.80038745784601,
127
+ "lr_score": 90.60055966133314,
128
+ "lr_calls": 84.34096290449882,
129
+ "rescued": 2163.4,
130
+ "harmed": 686.4
131
+ },
132
+ {
133
+ "key": [
134
+ "nlvr2",
135
+ "vilt",
136
+ "MiniCPM"
137
+ ],
138
+ "confidence_score": 90.64361053311329,
139
+ "confidence_calls": 95.93169261677549,
140
+ "lr_score": 90.69383655019013,
141
+ "lr_calls": 98.05984071177441,
142
+ "rescued": 2693.8,
143
+ "harmed": 704.8
144
+ },
145
+ {
146
+ "key": [
147
+ "nlvr2",
148
+ "vilt",
149
+ "Qwen3.8"
150
+ ],
151
+ "confidence_score": 89.96197172992753,
152
+ "confidence_calls": 95.93169261677549,
153
+ "lr_score": 90.0652938221999,
154
+ "lr_calls": 99.61684724115663,
155
+ "rescued": 2788.2,
156
+ "harmed": 886.8
157
+ },
158
+ {
159
+ "key": [
160
+ "construction",
161
+ "yolo26x",
162
+ "MiniCPM"
163
+ ],
164
+ "confidence_score": 25.23047077304935,
165
+ "confidence_calls": 92.54327563249002,
166
+ "lr_score": 25.880436219165098,
167
+ "lr_calls": 45.31957390146471,
168
+ "rescued": 1139.6,
169
+ "harmed": 803.4
170
+ },
171
+ {
172
+ "key": [
173
+ "construction",
174
+ "yolo26x",
175
+ "Qwen3.8"
176
+ ],
177
+ "confidence_score": 25.974900047621052,
178
+ "confidence_calls": 92.21038615179761,
179
+ "lr_score": 29.79678330127048,
180
+ "lr_calls": 44.766977363515316,
181
+ "rescued": 1231.6,
182
+ "harmed": 315.0
183
+ },
184
+ {
185
+ "key": [
186
+ "construction",
187
+ "rtdetr_x_fp32",
188
+ "MiniCPM"
189
+ ],
190
+ "confidence_score": 22.361115727328066,
191
+ "confidence_calls": 100.0,
192
+ "lr_score": 22.79931433315267,
193
+ "lr_calls": 78.43541944074566,
194
+ "rescued": 3288.6,
195
+ "harmed": 1452.4
196
+ },
197
+ {
198
+ "key": [
199
+ "construction",
200
+ "rtdetr_x_fp32",
201
+ "Qwen3.8"
202
+ ],
203
+ "confidence_score": 27.001009067886244,
204
+ "confidence_calls": 100.0,
205
+ "lr_score": 26.783631605767233,
206
+ "lr_calls": 83.56857523302264,
207
+ "rescued": 3448.0,
208
+ "harmed": 574.8
209
+ }
210
+ ]
releases/20260913_lr/paper_data/main_table_cost_20260912/build_figures.py ADDED
@@ -0,0 +1,101 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Main-table curves plus fresh measured component costs; no threshold re-selection."""
2
+ from pathlib import Path
3
+ import csv
4
+ import json
5
+ import numpy as np
6
+ import matplotlib
7
+ matplotlib.use('Agg')
8
+ import matplotlib.pyplot as plt
9
+ from matplotlib.ticker import MaxNLocator,LogLocator,NullLocator
10
+
11
+ root=Path(__file__).resolve().parent
12
+ groups=json.loads((root/'main_table_results.json').read_text(encoding='utf-8'))['groups']
13
+ components={}
14
+ curves=[]
15
+ selected=[]
16
+ for g,tag in zip(groups,['minicpm','qwen']):
17
+ expert=g['key'][1]
18
+ spec=json.loads((root/f'components/{expert}_specialist.json').read_text())
19
+ router=json.loads((root/f'components/{expert}_router.json').read_text())
20
+ large=json.loads((root/f'components/{tag}_latency.json').read_text())
21
+ flops=json.loads((root/f'components/{tag}_flops.json').read_text())
22
+ assert spec['completed'] and router['status']==large['status']==flops['status']=='complete'
23
+ assert router['run']==g['run'] and router['selected_decisions_match_main']
24
+ assert abs(router['actual_call_fraction']-g['methods']['learned']['test_point']['actual_fraction'])<1e-12
25
+ key='|'.join(g['key'])
26
+ c={'specialist_gflops':spec['mean_counted_gflops_per_sample'],
27
+ 'router_gflops':router['mean_gflops_per_input'],
28
+ 'large_gflops':flops['mean_prefill_gflops_counted']+large['mean_completion_tokens']*flops['mean_decode_one_token_gflops_counted'],
29
+ 'specialist_ms':spec['mean_end_to_end_latency_ms_per_sample'],
30
+ 'router_ms':router['mean_latency_ms_per_input'],
31
+ 'large_ms':large['mean_batch_wall_ms_per_sample'],
32
+ 'large_samples':large['measured_count'], 'flops_probes':flops['sample_count'],
33
+ 'specialist_samples':spec['sample_count'],'router_samples':router['sample_count'],'batch_size':large['batch_size']}
34
+ components[key]=c
35
+ for method in ['learned','confidence']:
36
+ for kind,points in [('curve',g['methods'][method]['test_curve']),('selected',[g['methods'][method]['test_point']])]:
37
+ for p in points:
38
+ rate=p['actual_fraction']
39
+ row={'pair':key,'method':method,'nominal_validation_budget':p['budget'],'actual_test_call_fraction':rate,
40
+ 'test_metric':p['metric'],'estimated_gflops_per_input':c['specialist_gflops']+(c['router_gflops'] if method=='learned' else 0)+rate*c['large_gflops'],
41
+ 'estimated_time_ms_per_input':c['specialist_ms']+(c['router_ms'] if method=='learned' else 0)+rate*c['large_ms']}
42
+ (curves if kind=='curve' else selected).append(row)
43
+ for name,rows in [('full_cost_performance_curves',curves),('selected_points',selected),('fixed_budget_cost_performance',[p for p in curves if p['nominal_validation_budget'] in [0,.05,.1,.25,.5,.75,1]])]:
44
+ with (root/(name+'.csv')).open('w',encoding='utf-8',newline='') as f:
45
+ writer=csv.DictWriter(f,fieldnames=list(rows[0]));writer.writeheader();writer.writerows(rows)
46
+ summary={'components':components,'selected_points':selected,'protocol':
47
+ {'method':'original main-table logit contrast; original checkpoints and validation thresholds; seed 42',
48
+ 'large_timing':'64 measured examples; batch size one; same full task prompt; two separate warmup examples',
49
+ 'cost':'serial component estimates, not a timed integrated service; includes active distilled encoders and routing head only for learned',
50
+ 'flops':'multiply-add=2; supported operators; mean prefill (including visual embedding) plus average generated length times one-token decode probe',
51
+ 'scope_limits':'mean call cost extrapolated across policies; no policy-specific input/output-length costs; specialist-feature extraction overhead is not separately measured; confidence threshold overhead not separately timed'}}
52
+ (root/'cost_summary.json').write_text(json.dumps(summary,indent=2),encoding='utf-8')
53
+
54
+ plt.rcParams.update({'font.family':'serif','font.serif':['Times New Roman'],'font.size':7,
55
+ 'axes.labelsize':7,'xtick.labelsize':6.5,'ytick.labelsize':6.5,'legend.fontsize':7,
56
+ 'pdf.fonttype':42,'ps.fonttype':42,'axes.spines.top':False,'axes.spines.right':False,
57
+ 'axes.linewidth':.55,'mathtext.fontset':'stix'})
58
+ styles=[('learned','learned','#176f91','-'),('confidence','conf.','#c78324','--')]
59
+ keys=list(components)
60
+
61
+ def panel(ax,key,field):
62
+ for method,label,color,ls in styles:
63
+ pts=[p for p in curves if p['pair']==key and p['method']==method]
64
+ assert len(pts)==101
65
+ x=np.array([p[field] for p in pts]);y=np.array([p['test_metric']*100 for p in pts])
66
+ assert np.all(np.diff(x)>=-1e-7)
67
+ ax.plot(x,y,color=color,ls=ls,lw=1.05,label=label)
68
+ p=next(p for p in selected if p['pair']==key and p['method']==method)
69
+ ax.scatter(p[field],100*p['test_metric'],color=color,s=13,edgecolor='white',linewidth=.4,zorder=5)
70
+ ax.grid(axis='y',color='#dce1e4',lw=.4)
71
+ ax.tick_params(length=2.3,pad=1.5)
72
+ ax.yaxis.set_major_locator(MaxNLocator(nbins=3))
73
+ if 'gflops' in field:
74
+ ax.set_xscale('log');ax.xaxis.set_major_locator(LogLocator(base=10,numticks=4));ax.xaxis.set_minor_locator(NullLocator())
75
+ ax.set_xlabel('Est. GFLOPs/input',labelpad=1)
76
+ else:
77
+ ax.xaxis.set_major_locator(MaxNLocator(nbins=3));ax.set_xlabel('Est. time (ms/input)',labelpad=1)
78
+ ax.set_ylabel('Acc. (%)' if key.startswith('grefcoco') else 'Macro-F1 (%)',labelpad=1)
79
+
80
+ for field,stem in [('estimated_gflops_per_input','flops_performance'),('estimated_time_ms_per_input','latency_performance')]:
81
+ fig,axes=plt.subplots(1,2,figsize=(3.44,1.50))
82
+ fig.subplots_adjust(left=.115,right=.995,bottom=.31,top=.83,wspace=.43)
83
+ for ax,key,title in zip(axes,keys,['gRefCOCO / M','Construction / Q']):
84
+ panel(ax,key,field)
85
+ ax.text(.5,1.10,title,transform=ax.transAxes,ha='center',fontsize=7)
86
+ h,l=axes[0].get_legend_handles_labels()
87
+ fig.legend(h,l,loc='lower center',bbox_to_anchor=(.53,-.025),ncol=2,frameon=False,handlelength=2.2,columnspacing=1.8)
88
+ fig.savefig(root/(stem+'.pdf'));fig.savefig(root/(stem+'.png'),dpi=400)
89
+ plt.close(fig)
90
+
91
+ fig,axes=plt.subplots(2,2,figsize=(3.44,2.05))
92
+ fig.subplots_adjust(left=.115,right=.995,bottom=.20,top=.89,wspace=.43,hspace=.64)
93
+ for col,key in enumerate(keys):
94
+ for row,field in enumerate(['estimated_gflops_per_input','estimated_time_ms_per_input']):
95
+ panel(axes[row,col],key,field)
96
+ axes[0,col].text(.5,1.14,'gRefCOCO / M' if col==0 else 'Construction / Q',transform=axes[0,col].transAxes,ha='center',fontsize=7)
97
+ h,l=axes[0,0].get_legend_handles_labels()
98
+ fig.legend(h,l,loc='lower center',bbox_to_anchor=(.53,-.005),ncol=2,frameon=False,handlelength=2.2,columnspacing=1.8)
99
+ fig.savefig(root/'cost_compact.pdf');fig.savefig(root/'cost_compact.png',dpi=400)
100
+ plt.close(fig)
101
+ print(json.dumps(summary,indent=2))
releases/20260913_lr/paper_data/main_table_cost_20260912/components/instancevg_router.json ADDED
@@ -0,0 +1,239 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "complete",
3
+ "run": "shared_12_instancevg_MiniCPM",
4
+ "config": {
5
+ "fraction": 1.0,
6
+ "seed": 42,
7
+ "freeze_encoder": true,
8
+ "fusion": "concat",
9
+ "harm_cost": 1.0,
10
+ "batch_size": 256,
11
+ "weight_decay": 0.01,
12
+ "max_epochs": 50,
13
+ "patience": 8,
14
+ "cpu_threads": 1,
15
+ "student": "8M",
16
+ "representation_dim": 128,
17
+ "lr": 0.003,
18
+ "target": "four_state",
19
+ "strategy": "natural",
20
+ "hidden_percent": 90,
21
+ "expert": "instancevg",
22
+ "task": "grefcoco",
23
+ "endpoint": "MiniCPM",
24
+ "branches": [
25
+ "image",
26
+ "text",
27
+ "hidden",
28
+ "confidence",
29
+ "output"
30
+ ],
31
+ "hidden_layer": "joint_encoder__layer_11",
32
+ "id": "shared_12_instancevg_MiniCPM",
33
+ "stage": "shared_mandatory_inputs"
34
+ },
35
+ "checkpoint": "/mnt/HithinkOmniSSD/user_workspace/caisihang/research/ICASSP_ModelCollaboration/outputs/router_shared_image_hidden_20260910/runs/shared_12_instancevg_MiniCPM/best.pt",
36
+ "score": "logit_rescue-logit_harm",
37
+ "test_rows_recomputed": 35263,
38
+ "selected_decisions_match_main": true,
39
+ "max_score_abs_difference": 0.0,
40
+ "actual_call_fraction": 0.30904914499617164,
41
+ "sample_count": 64,
42
+ "sample_ids": [
43
+ "testB-5056-14012",
44
+ "testB-31505-86708",
45
+ "testA-57489-158264",
46
+ "testA-38865-107108",
47
+ "testB-47361-130361",
48
+ "testA-5476-15123",
49
+ "testA-71404-196892",
50
+ "testA-61030-168115",
51
+ "testB-35969-99063",
52
+ "testB-1675-4724",
53
+ "testA-30414-83641",
54
+ "testB-47742-131425",
55
+ "testB-77456-213280",
56
+ "testA-59283-163253",
57
+ "testA-57390-157996",
58
+ "testA-77229-212665",
59
+ "testA-6784-18703",
60
+ "testB-11434-31542",
61
+ "testB-11675-32181",
62
+ "testB-73046-201304",
63
+ "testA-28837-79356",
64
+ "testA-4563-12619",
65
+ "testB-30168-82996",
66
+ "testB-92461-254721",
67
+ "testA-58293-160462",
68
+ "testA-13050-35910",
69
+ "testA-39320-108309",
70
+ "testB-36423-100313",
71
+ "testA-43418-119528",
72
+ "testB-23411-64351",
73
+ "testB-77730-214031",
74
+ "testA-7232-19907",
75
+ "testA-17738-48695",
76
+ "testA-44914-123593",
77
+ "testB-92861-255854",
78
+ "testA-69678-192132",
79
+ "testA-68785-189658",
80
+ "testB-62911-173380",
81
+ "testB-19029-52242",
82
+ "testA-66284-182759",
83
+ "testB-19170-52626",
84
+ "testB-8468-23291",
85
+ "testB-72940-201009",
86
+ "testB-74045-203983",
87
+ "testB-93380-257275",
88
+ "testB-49369-135990",
89
+ "testA-58046-159759",
90
+ "testA-57437-158132",
91
+ "testA-5754-15895",
92
+ "testA-85590-235577",
93
+ "testA-92278-254227",
94
+ "testB-80373-221257",
95
+ "testA-80383-221294",
96
+ "testB-71197-196311",
97
+ "testB-48080-132352",
98
+ "testB-18351-50380",
99
+ "testB-1089-3077",
100
+ "testB-92466-254737",
101
+ "testB-53065-146099",
102
+ "testB-42611-117349",
103
+ "testA-57690-158796",
104
+ "testA-65173-179681",
105
+ "testB-90675-249707",
106
+ "testB-74709-205815"
107
+ ],
108
+ "batch_size": 1,
109
+ "latency_ms": [
110
+ 13.662165962159634,
111
+ 9.513642988167703,
112
+ 8.86400998570025,
113
+ 7.89595500100404,
114
+ 9.713676990941167,
115
+ 9.162381989881396,
116
+ 9.548410074785352,
117
+ 9.09417902585119,
118
+ 11.833172990009189,
119
+ 14.292752020992339,
120
+ 10.746326064690948,
121
+ 10.920883971266448,
122
+ 19.845983013510704,
123
+ 14.003381016664207,
124
+ 15.885940054431558,
125
+ 14.8980199592188,
126
+ 15.25757904164493,
127
+ 17.458743997849524,
128
+ 17.598357051610947,
129
+ 15.274635050445795,
130
+ 15.84710506722331,
131
+ 31.497937976382673,
132
+ 17.706500948406756,
133
+ 13.621860067360103,
134
+ 12.55292899440974,
135
+ 10.675732046365738,
136
+ 9.522471926175058,
137
+ 11.19849900715053,
138
+ 13.095670961774886,
139
+ 10.178428026847541,
140
+ 11.139138019643724,
141
+ 10.480726952664554,
142
+ 9.46063396986574,
143
+ 10.675741010345519,
144
+ 9.826205903664231,
145
+ 9.403666015714407,
146
+ 10.096908081322908,
147
+ 28.78253604285419,
148
+ 21.029149065725505,
149
+ 10.848066071048379,
150
+ 10.954314027912915,
151
+ 11.451196041889489,
152
+ 11.626868974417448,
153
+ 9.91171004716307,
154
+ 12.449359055608511,
155
+ 11.346090934239328,
156
+ 10.699236067011952,
157
+ 10.140218073502183,
158
+ 12.354035046882927,
159
+ 14.74389701616019,
160
+ 11.097895912826061,
161
+ 11.268700007349253,
162
+ 12.365077971480787,
163
+ 10.821785079315305,
164
+ 11.905013001523912,
165
+ 25.439211982302368,
166
+ 15.189048950560391,
167
+ 12.554107001051307,
168
+ 10.584128089249134,
169
+ 23.863457958213985,
170
+ 14.576193992979825,
171
+ 11.179295019246638,
172
+ 12.35828804783523,
173
+ 11.567718931473792
174
+ ],
175
+ "mean_latency_ms_per_input": 13.180577338061994,
176
+ "head_mean_latency_ms": 0.6150530008599162,
177
+ "gflops_per_input": [
178
+ 3.642171392,
179
+ 3.642171392,
180
+ 3.642171392,
181
+ 3.642171392,
182
+ 3.642171392,
183
+ 3.642171392,
184
+ 3.642171392,
185
+ 3.642171392
186
+ ],
187
+ "mean_gflops_per_input": 3.642171392,
188
+ "operators": [
189
+ {
190
+ "aten.convolution": 228802560,
191
+ "aten.bmm": 691709440,
192
+ "aten._scaled_dot_product_flash_attention": 236027904,
193
+ "aten.addmm": 2485631488
194
+ },
195
+ {
196
+ "aten.convolution": 228802560,
197
+ "aten.bmm": 691709440,
198
+ "aten._scaled_dot_product_flash_attention": 236027904,
199
+ "aten.addmm": 2485631488
200
+ },
201
+ {
202
+ "aten.convolution": 228802560,
203
+ "aten.bmm": 691709440,
204
+ "aten._scaled_dot_product_flash_attention": 236027904,
205
+ "aten.addmm": 2485631488
206
+ },
207
+ {
208
+ "aten.convolution": 228802560,
209
+ "aten.bmm": 691709440,
210
+ "aten._scaled_dot_product_flash_attention": 236027904,
211
+ "aten.addmm": 2485631488
212
+ },
213
+ {
214
+ "aten.convolution": 228802560,
215
+ "aten.bmm": 691709440,
216
+ "aten._scaled_dot_product_flash_attention": 236027904,
217
+ "aten.addmm": 2485631488
218
+ },
219
+ {
220
+ "aten.convolution": 228802560,
221
+ "aten.bmm": 691709440,
222
+ "aten._scaled_dot_product_flash_attention": 236027904,
223
+ "aten.addmm": 2485631488
224
+ },
225
+ {
226
+ "aten.convolution": 228802560,
227
+ "aten.bmm": 691709440,
228
+ "aten._scaled_dot_product_flash_attention": 236027904,
229
+ "aten.addmm": 2485631488
230
+ },
231
+ {
232
+ "aten.convolution": 228802560,
233
+ "aten.bmm": 691709440,
234
+ "aten._scaled_dot_product_flash_attention": 236027904,
235
+ "aten.addmm": 2485631488
236
+ }
237
+ ],
238
+ "scope": "image loading, bilinear resize, tokenize when used, two image slots (including gray placeholder), active distilled encoders, feature assembly and original routing head; cached specialist features; supported FLOPs only"
239
+ }
releases/20260913_lr/paper_data/main_table_cost_20260912/components/instancevg_specialist.json ADDED
@@ -0,0 +1,254 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "component": "specialist",
3
+ "expert": "instancevg",
4
+ "task": "grefcoco",
5
+ "device": "cuda:0",
6
+ "checkpoint": "/mnt/HithinkOmniSSD/user_workspace/caisihang/research/ICASSP_ModelCollaboration/models/grefcoco/instancevg/InstanceVG-grefcoco.pth",
7
+ "sample_count": 64,
8
+ "batch_size": 1,
9
+ "sample_ids": [
10
+ "testA-30844-84861",
11
+ "testB-44353-122077",
12
+ "testB-36889-101643",
13
+ "testA-49055-135087",
14
+ "testA-81245-223633",
15
+ "testB-37669-103833",
16
+ "testA-5038-13943",
17
+ "testB-79646-219274",
18
+ "testA-469-1309",
19
+ "testB-46644-128438",
20
+ "testA-24562-67515",
21
+ "testA-47070-129599",
22
+ "testA-54554-150141",
23
+ "testB-12382-34099",
24
+ "testB-23014-63279",
25
+ "testB-79647-219275",
26
+ "testB-69479-191564",
27
+ "testA-15582-42797",
28
+ "testB-56692-156000",
29
+ "testB-42726-117660",
30
+ "testA-80455-221494",
31
+ "testB-33115-91174",
32
+ "testA-62940-173455",
33
+ "testB-31494-86670",
34
+ "testA-54586-150233",
35
+ "testA-81445-224177",
36
+ "testB-8417-23141",
37
+ "testA-24438-67190",
38
+ "testA-59285-163255",
39
+ "testA-73258-201893",
40
+ "testB-945-2633",
41
+ "testA-54307-149426",
42
+ "testB-60557-166822",
43
+ "testA-81622-224665",
44
+ "testA-69466-191525",
45
+ "testA-60731-167321",
46
+ "testA-7857-21644",
47
+ "testA-44142-121499",
48
+ "testB-66162-182407",
49
+ "testA-31764-87406",
50
+ "testB-38637-106482",
51
+ "testB-52921-145709",
52
+ "testB-65796-181394",
53
+ "testB-2936-8113",
54
+ "testA-79438-218690",
55
+ "testA-57397-158016",
56
+ "testB-23920-65743",
57
+ "testB-15419-42358",
58
+ "testA-10078-27739",
59
+ "testA-43890-120793",
60
+ "testA-55116-151689",
61
+ "testB-27487-75582",
62
+ "testA-26856-73830",
63
+ "testB-50845-140107",
64
+ "testA-29199-80374",
65
+ "testB-65935-181778",
66
+ "testA-17736-48684",
67
+ "testB-69477-191553",
68
+ "testB-22290-61227",
69
+ "testA-58476-160950",
70
+ "testB-36265-99859",
71
+ "testA-5734-15837",
72
+ "testB-67952-187353",
73
+ "testB-47054-129555"
74
+ ],
75
+ "model_load_seconds_excluded_from_latency": true,
76
+ "forward_latency_ms_per_sample": [
77
+ 41.24195098876953,
78
+ 41.74086380004883,
79
+ 40.36540985107422,
80
+ 44.46102523803711,
81
+ 41.2421760559082,
82
+ 40.76591873168945,
83
+ 41.14704132080078,
84
+ 40.88172912597656,
85
+ 41.70716857910156,
86
+ 41.191871643066406,
87
+ 39.91231918334961,
88
+ 39.29497528076172,
89
+ 40.70723342895508,
90
+ 38.920894622802734,
91
+ 39.32780838012695,
92
+ 38.89958572387695,
93
+ 38.59721755981445,
94
+ 38.47331237792969,
95
+ 39.8744010925293,
96
+ 39.87260818481445,
97
+ 41.48284912109375,
98
+ 40.55779266357422,
99
+ 39.05420684814453,
100
+ 39.052734375,
101
+ 39.143775939941406,
102
+ 39.82249450683594,
103
+ 39.543006896972656,
104
+ 39.368160247802734,
105
+ 40.661216735839844,
106
+ 39.092193603515625,
107
+ 39.85852813720703,
108
+ 40.33164978027344,
109
+ 38.89593505859375,
110
+ 39.29216003417969,
111
+ 39.20943832397461,
112
+ 40.946590423583984,
113
+ 39.44643020629883,
114
+ 40.220863342285156,
115
+ 39.61030578613281,
116
+ 39.639007568359375,
117
+ 39.78163146972656,
118
+ 39.223201751708984,
119
+ 39.00921630859375,
120
+ 38.99894332885742,
121
+ 41.13721466064453,
122
+ 39.94972610473633,
123
+ 41.41766357421875,
124
+ 39.28073501586914,
125
+ 39.721214294433594,
126
+ 39.24937438964844,
127
+ 39.40460968017578,
128
+ 41.30179214477539,
129
+ 42.94054412841797,
130
+ 41.5063362121582,
131
+ 44.03267288208008,
132
+ 44.55561447143555,
133
+ 44.24995040893555,
134
+ 44.87868881225586,
135
+ 45.14652633666992,
136
+ 44.85055923461914,
137
+ 42.39228820800781,
138
+ 41.121856689453125,
139
+ 45.708351135253906,
140
+ 41.8752326965332
141
+ ],
142
+ "end_to_end_latency_ms_per_sample": [
143
+ 47.29630902875215,
144
+ 46.27182998228818,
145
+ 47.56885801907629,
146
+ 45.48592597711831,
147
+ 45.399176073260605,
148
+ 46.442538034170866,
149
+ 45.95420998521149,
150
+ 46.45739798434079,
151
+ 44.90085388533771,
152
+ 43.80481690168381,
153
+ 44.848884106613696,
154
+ 43.67590101901442,
155
+ 46.40063806436956,
156
+ 44.26892101764679,
157
+ 44.44277100265026,
158
+ 44.71806308720261,
159
+ 44.62323593907058,
160
+ 44.6264409692958,
161
+ 44.96095096692443,
162
+ 44.103492982685566,
163
+ 45.13338196557015,
164
+ 44.1846780013293,
165
+ 48.1606520479545,
166
+ 46.43543600104749,
167
+ 46.19023704435676,
168
+ 45.247678994201124,
169
+ 44.801001087762415,
170
+ 45.467636082321405,
171
+ 44.22409099061042,
172
+ 44.7448689956218,
173
+ 58.0457029864192,
174
+ 46.10048606991768,
175
+ 44.56539999227971,
176
+ 44.47658592835069,
177
+ 46.05250502936542,
178
+ 44.743441976606846,
179
+ 44.77924096863717,
180
+ 43.877715012058616,
181
+ 44.05005706939846,
182
+ 42.87725198082626,
183
+ 43.836933095008135,
184
+ 45.634721987880766,
185
+ 44.748114072717726,
186
+ 46.21323198080063,
187
+ 52.24333005025983,
188
+ 48.90036000870168,
189
+ 48.12136595137417,
190
+ 47.65231907367706,
191
+ 46.71858798246831,
192
+ 48.516221111640334,
193
+ 43.90157596208155,
194
+ 46.83465394191444,
195
+ 46.49809293914586,
196
+ 42.6300170365721,
197
+ 47.39421408157796,
198
+ 48.35387098137289,
199
+ 48.176085110753775,
200
+ 48.29539393540472,
201
+ 47.32040502130985,
202
+ 46.35261802468449,
203
+ 47.482443042099476,
204
+ 47.437409055419266,
205
+ 47.27676196489483,
206
+ 48.198972013778985
207
+ ],
208
+ "mean_forward_latency_ms_per_sample": 40.71232491731644,
209
+ "median_forward_latency_ms_per_sample": 40.08529472351074,
210
+ "mean_end_to_end_latency_ms_per_sample": 46.08042127620138,
211
+ "median_end_to_end_latency_ms_per_sample": 45.79446598654613,
212
+ "flop_profiles": [
213
+ {
214
+ "batch_size": 1,
215
+ "flops_counted": 7502680064,
216
+ "gflops_counted_per_sample": 7.502680064,
217
+ "operators": {
218
+ "aten.bmm": 7500632064,
219
+ "aten.baddbmm": 2048000
220
+ }
221
+ },
222
+ {
223
+ "batch_size": 1,
224
+ "flops_counted": 7502680064,
225
+ "gflops_counted_per_sample": 7.502680064,
226
+ "operators": {
227
+ "aten.bmm": 7500632064,
228
+ "aten.baddbmm": 2048000
229
+ }
230
+ },
231
+ {
232
+ "batch_size": 1,
233
+ "flops_counted": 7502680064,
234
+ "gflops_counted_per_sample": 7.502680064,
235
+ "operators": {
236
+ "aten.bmm": 7500632064,
237
+ "aten.baddbmm": 2048000
238
+ }
239
+ },
240
+ {
241
+ "batch_size": 1,
242
+ "flops_counted": 7502680064,
243
+ "gflops_counted_per_sample": 7.502680064,
244
+ "operators": {
245
+ "aten.bmm": 7500632064,
246
+ "aten.baddbmm": 2048000
247
+ }
248
+ }
249
+ ],
250
+ "mean_counted_gflops_per_sample": 7.502680064,
251
+ "flops_scope": "GPU FlopCounterMode dense products, convolutions and supported attention; multiply-add counted as 2 FLOPs. Elementwise, normalization, custom ops and postprocessing may be missing.",
252
+ "latency_scope": "A800 wall-clock CUDA forward latency and end-to-end latency including image read/preprocess, excluding model load; batch processing uses the recorded batch size.",
253
+ "completed": true
254
+ }
releases/20260913_lr/paper_data/main_table_cost_20260912/components/minicpm_flops.json ADDED
@@ -0,0 +1,261 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "complete",
3
+ "model_kind": "MiniCPM",
4
+ "task": "grefcoco",
5
+ "model_path": "/mnt/HithinkOmniSSD/user_workspace/caisihang/checkpoints/MiniCPM-V-4_5",
6
+ "sample_count": 8,
7
+ "records": [
8
+ {
9
+ "sample_id": "testA-5038-13945",
10
+ "prompt_tokens_in_reference_tensor": 528,
11
+ "input_shapes": {
12
+ "input_ids": [
13
+ 1,
14
+ 528
15
+ ],
16
+ "attention_mask": [
17
+ 1,
18
+ 528
19
+ ]
20
+ },
21
+ "prefill_flops_counted": 10720928684032,
22
+ "decode_one_token_flops_counted": 15446671488,
23
+ "prefill_gflops_counted": 10720.928684032,
24
+ "decode_one_token_gflops_counted": 15.446671488,
25
+ "prefill_operators": {
26
+ "aten.bmm": 164433561600,
27
+ "aten.mm": 7335973519360,
28
+ "vision:aten.convolution": 4210569216,
29
+ "vision:aten.addmm": 2568098906112,
30
+ "vision:aten.bmm": 610809421824,
31
+ "vision:aten.mm": 35773218816,
32
+ "vision:aten.baddbmm": 1629487104
33
+ },
34
+ "decode_operators": {
35
+ "aten.bmm": 312017024,
36
+ "aten.mm": 15134654464
37
+ }
38
+ },
39
+ {
40
+ "sample_id": "testA-73305-202014",
41
+ "prompt_tokens_in_reference_tensor": 528,
42
+ "input_shapes": {
43
+ "input_ids": [
44
+ 1,
45
+ 528
46
+ ],
47
+ "attention_mask": [
48
+ 1,
49
+ 528
50
+ ]
51
+ },
52
+ "prefill_flops_counted": 10720928684032,
53
+ "decode_one_token_flops_counted": 15446671488,
54
+ "prefill_gflops_counted": 10720.928684032,
55
+ "decode_one_token_gflops_counted": 15.446671488,
56
+ "prefill_operators": {
57
+ "aten.bmm": 164433561600,
58
+ "aten.mm": 7335973519360,
59
+ "vision:aten.convolution": 4210569216,
60
+ "vision:aten.addmm": 2568098906112,
61
+ "vision:aten.bmm": 610809421824,
62
+ "vision:aten.mm": 35773218816,
63
+ "vision:aten.baddbmm": 1629487104
64
+ },
65
+ "decode_operators": {
66
+ "aten.bmm": 312017024,
67
+ "aten.mm": 15134654464
68
+ }
69
+ },
70
+ {
71
+ "sample_id": "testA-26952-74099",
72
+ "prompt_tokens_in_reference_tensor": 523,
73
+ "input_shapes": {
74
+ "input_ids": [
75
+ 1,
76
+ 523
77
+ ],
78
+ "attention_mask": [
79
+ 1,
80
+ 523
81
+ ]
82
+ },
83
+ "prefill_flops_counted": 10648371484032,
84
+ "decode_one_token_flops_counted": 15443722368,
85
+ "prefill_gflops_counted": 10648.371484032,
86
+ "decode_one_token_gflops_counted": 15.443722368,
87
+ "prefill_operators": {
88
+ "aten.bmm": 161334035840,
89
+ "aten.mm": 7266515845120,
90
+ "vision:aten.convolution": 4210569216,
91
+ "vision:aten.addmm": 2568098906112,
92
+ "vision:aten.bmm": 610809421824,
93
+ "vision:aten.mm": 35773218816,
94
+ "vision:aten.baddbmm": 1629487104
95
+ },
96
+ "decode_operators": {
97
+ "aten.bmm": 309067904,
98
+ "aten.mm": 15134654464
99
+ }
100
+ },
101
+ {
102
+ "sample_id": "testB-53254-146620",
103
+ "prompt_tokens_in_reference_tensor": 520,
104
+ "input_shapes": {
105
+ "input_ids": [
106
+ 1,
107
+ 520
108
+ ],
109
+ "attention_mask": [
110
+ 1,
111
+ 520
112
+ ]
113
+ },
114
+ "prefill_flops_counted": 10604851319808,
115
+ "decode_one_token_flops_counted": 15441952896,
116
+ "prefill_gflops_counted": 10604.851319808,
117
+ "decode_one_token_gflops_counted": 15.441952896,
118
+ "prefill_operators": {
119
+ "aten.bmm": 159488476160,
120
+ "aten.mm": 7224841240576,
121
+ "vision:aten.convolution": 4210569216,
122
+ "vision:aten.addmm": 2568098906112,
123
+ "vision:aten.bmm": 610809421824,
124
+ "vision:aten.mm": 35773218816,
125
+ "vision:aten.baddbmm": 1629487104
126
+ },
127
+ "decode_operators": {
128
+ "aten.bmm": 307298432,
129
+ "aten.mm": 15134654464
130
+ }
131
+ },
132
+ {
133
+ "sample_id": "testB-37889-104439",
134
+ "prompt_tokens_in_reference_tensor": 528,
135
+ "input_shapes": {
136
+ "input_ids": [
137
+ 1,
138
+ 528
139
+ ],
140
+ "attention_mask": [
141
+ 1,
142
+ 528
143
+ ]
144
+ },
145
+ "prefill_flops_counted": 10637704296448,
146
+ "decode_one_token_flops_counted": 15446671488,
147
+ "prefill_gflops_counted": 10637.704296448,
148
+ "decode_one_token_gflops_counted": 15.446671488,
149
+ "prefill_operators": {
150
+ "aten.bmm": 164433561600,
151
+ "aten.mm": 7335973519360,
152
+ "vision:aten.convolution": 4113027072,
153
+ "vision:aten.addmm": 2508904759296,
154
+ "vision:aten.bmm": 587593949184,
155
+ "vision:aten.mm": 35093741568,
156
+ "vision:aten.baddbmm": 1591738368
157
+ },
158
+ "decode_operators": {
159
+ "aten.bmm": 312017024,
160
+ "aten.mm": 15134654464
161
+ }
162
+ },
163
+ {
164
+ "sample_id": "testA-58523-161084",
165
+ "prompt_tokens_in_reference_tensor": 396,
166
+ "input_shapes": {
167
+ "input_ids": [
168
+ 1,
169
+ 396
170
+ ],
171
+ "attention_mask": [
172
+ 1,
173
+ 396
174
+ ]
175
+ },
176
+ "prefill_flops_counted": 6668292011520,
177
+ "decode_one_token_flops_counted": 15368814720,
178
+ "prefill_gflops_counted": 6668.29201152,
179
+ "decode_one_token_gflops_counted": 15.36881472,
180
+ "prefill_operators": {
181
+ "aten.bmm": 92493891072,
182
+ "aten.mm": 5502290919424,
183
+ "vision:aten.convolution": 1403523072,
184
+ "vision:aten.addmm": 925557751808,
185
+ "vision:aten.bmm": 134078357504,
186
+ "vision:aten.mm": 11924406272,
187
+ "vision:aten.baddbmm": 543162368
188
+ },
189
+ "decode_operators": {
190
+ "aten.bmm": 234160256,
191
+ "aten.mm": 15134654464
192
+ }
193
+ },
194
+ {
195
+ "sample_id": "testB-3419-9449",
196
+ "prompt_tokens_in_reference_tensor": 528,
197
+ "input_shapes": {
198
+ "input_ids": [
199
+ 1,
200
+ 528
201
+ ],
202
+ "attention_mask": [
203
+ 1,
204
+ 528
205
+ ]
206
+ },
207
+ "prefill_flops_counted": 10720928684032,
208
+ "decode_one_token_flops_counted": 15446671488,
209
+ "prefill_gflops_counted": 10720.928684032,
210
+ "decode_one_token_gflops_counted": 15.446671488,
211
+ "prefill_operators": {
212
+ "aten.bmm": 164433561600,
213
+ "aten.mm": 7335973519360,
214
+ "vision:aten.convolution": 4210569216,
215
+ "vision:aten.addmm": 2568098906112,
216
+ "vision:aten.bmm": 610809421824,
217
+ "vision:aten.mm": 35773218816,
218
+ "vision:aten.baddbmm": 1629487104
219
+ },
220
+ "decode_operators": {
221
+ "aten.bmm": 312017024,
222
+ "aten.mm": 15134654464
223
+ }
224
+ },
225
+ {
226
+ "sample_id": "testB-24035-66084",
227
+ "prompt_tokens_in_reference_tensor": 521,
228
+ "input_shapes": {
229
+ "input_ids": [
230
+ 1,
231
+ 521
232
+ ],
233
+ "attention_mask": [
234
+ 1,
235
+ 521
236
+ ]
237
+ },
238
+ "prefill_flops_counted": 10619356861568,
239
+ "decode_one_token_flops_counted": 15442542720,
240
+ "prefill_gflops_counted": 10619.356861568,
241
+ "decode_one_token_gflops_counted": 15.44254272,
242
+ "prefill_operators": {
243
+ "aten.bmm": 160102483072,
244
+ "aten.mm": 7238732775424,
245
+ "vision:aten.convolution": 4210569216,
246
+ "vision:aten.addmm": 2568098906112,
247
+ "vision:aten.bmm": 610809421824,
248
+ "vision:aten.mm": 35773218816,
249
+ "vision:aten.baddbmm": 1629487104
250
+ },
251
+ "decode_operators": {
252
+ "aten.bmm": 307888256,
253
+ "aten.mm": 15134654464
254
+ }
255
+ }
256
+ ],
257
+ "mean_prefill_gflops_counted": 10167.670253184,
258
+ "mean_decode_one_token_gflops_counted": 15.435464832,
259
+ "scope": "HF reference execution on the A800 with real task inputs; dense products/convolutions/supported attention counted with multiply-add=2 FLOPs. vLLM custom kernels, elementwise/normalization/nonlinearities and other unsupported operations are not complete.",
260
+ "answer_generation": "not performed; one known-token decode probe only"
261
+ }
releases/20260913_lr/paper_data/main_table_cost_20260912/components/minicpm_latency.json ADDED
@@ -0,0 +1,2105 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "complete",
3
+ "model_kind": "MiniCPM",
4
+ "task": "grefcoco",
5
+ "expert": "instancevg",
6
+ "model_path": "/mnt/HithinkOmniSSD/user_workspace/caisihang/checkpoints/MiniCPM-V-4_5",
7
+ "device": "cuda:0",
8
+ "gpu_memory_utilization": 0.85,
9
+ "sample_count_requested": 66,
10
+ "warmup_count": 2,
11
+ "measured_count": 64,
12
+ "batch_size": 1,
13
+ "sample_ids": [
14
+ "testA-29199-80373",
15
+ "testB-44351-122075",
16
+ "testA-91850-253019",
17
+ "testB-36888-101641",
18
+ "testA-54586-150233",
19
+ "testB-79644-219272",
20
+ "testB-65935-181776",
21
+ "testA-81622-224664",
22
+ "testA-10078-27739",
23
+ "testA-15582-42797",
24
+ "testA-5038-13943",
25
+ "testA-30844-84861",
26
+ "testB-15419-42356",
27
+ "testB-27486-75580",
28
+ "testB-945-2632",
29
+ "testA-24438-67190",
30
+ "testB-66161-182405",
31
+ "testB-42726-117659",
32
+ "testB-69439-191450",
33
+ "testA-54307-149426",
34
+ "testB-37669-103831",
35
+ "testB-50844-140105",
36
+ "testA-79438-218690",
37
+ "testA-62940-173454",
38
+ "testA-81245-223632",
39
+ "testA-47070-129599",
40
+ "testB-12381-34098",
41
+ "testA-44142-121498",
42
+ "testA-73258-201893",
43
+ "testB-46644-128436",
44
+ "testA-43889-120792",
45
+ "testB-36082-99363",
46
+ "testB-33108-91152",
47
+ "testA-26856-73830",
48
+ "testA-81271-223709",
49
+ "testA-54554-150141",
50
+ "testB-38636-106481",
51
+ "testA-5734-15837",
52
+ "testB-67558-186325",
53
+ "testB-47053-129554",
54
+ "testA-93064-256422",
55
+ "testB-60557-166821",
56
+ "testA-55116-151688",
57
+ "testB-65796-181392",
58
+ "testB-23014-63278",
59
+ "testA-24562-67515",
60
+ "testA-17736-48684",
61
+ "testA-58475-160949",
62
+ "testA-80455-221493",
63
+ "testA-31763-87405",
64
+ "testB-23920-65742",
65
+ "testA-469-1309",
66
+ "testB-56691-155998",
67
+ "testB-31493-86668",
68
+ "testA-59284-163254",
69
+ "testA-57397-158015",
70
+ "testB-2935-8111",
71
+ "testB-22290-61226",
72
+ "testB-8416-23140",
73
+ "testA-60731-167320",
74
+ "testA-49055-135086",
75
+ "testA-7857-21644",
76
+ "testB-79645-219273",
77
+ "testB-52921-145707",
78
+ "testB-69479-191562",
79
+ "testA-69466-191524"
80
+ ],
81
+ "batches": [
82
+ {
83
+ "sample_ids": [
84
+ "testA-91850-253019"
85
+ ],
86
+ "batch_size": 1,
87
+ "generation_latency_seconds": 0.8861644599819556,
88
+ "batch_wall_latency_seconds": 0.9864439839730039,
89
+ "amortized_generation_ms_per_sample": 886.1644599819556,
90
+ "amortized_batch_wall_ms_per_sample": 986.4439839730039,
91
+ "outputs": [
92
+ {
93
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[420 ,220 ,710 ,640 ],\"score\":0.95}]}",
94
+ "valid_prediction": {
95
+ "objects": [
96
+ {
97
+ "bbox_2d": [
98
+ 420,
99
+ 220,
100
+ 710,
101
+ 640
102
+ ],
103
+ "score": 0.95
104
+ }
105
+ ]
106
+ },
107
+ "error": null,
108
+ "finish_reason": "stop",
109
+ "completion_tokens": 39,
110
+ "prompt_tokens": 530
111
+ }
112
+ ]
113
+ },
114
+ {
115
+ "sample_ids": [
116
+ "testB-36888-101641"
117
+ ],
118
+ "batch_size": 1,
119
+ "generation_latency_seconds": 1.3824354230891913,
120
+ "batch_wall_latency_seconds": 1.3937634559115395,
121
+ "amortized_generation_ms_per_sample": 1382.4354230891913,
122
+ "amortized_batch_wall_ms_per_sample": 1393.7634559115395,
123
+ "outputs": [
124
+ {
125
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[30,570,720,900],\"score\":0.95},{\"bbox_2d\":[390,260,960,860],\"score\":0.93}]}",
126
+ "valid_prediction": {
127
+ "objects": [
128
+ {
129
+ "bbox_2d": [
130
+ 30,
131
+ 570,
132
+ 720,
133
+ 900
134
+ ],
135
+ "score": 0.95
136
+ },
137
+ {
138
+ "bbox_2d": [
139
+ 390,
140
+ 260,
141
+ 960,
142
+ 860
143
+ ],
144
+ "score": 0.93
145
+ }
146
+ ]
147
+ },
148
+ "error": null,
149
+ "finish_reason": "stop",
150
+ "completion_tokens": 60,
151
+ "prompt_tokens": 528
152
+ }
153
+ ]
154
+ },
155
+ {
156
+ "sample_ids": [
157
+ "testA-54586-150233"
158
+ ],
159
+ "batch_size": 1,
160
+ "generation_latency_seconds": 10.028413712978363,
161
+ "batch_wall_latency_seconds": 10.048457957920618,
162
+ "amortized_generation_ms_per_sample": 10028.413712978363,
163
+ "amortized_batch_wall_ms_per_sample": 10048.457957920618,
164
+ "outputs": [
165
+ {
166
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[480 ",
167
+ "valid_prediction": null,
168
+ "error": "Expecting ',' delimiter: line 1 column 529 (char 528)",
169
+ "finish_reason": "length",
170
+ "completion_tokens": 512,
171
+ "prompt_tokens": 660
172
+ }
173
+ ]
174
+ },
175
+ {
176
+ "sample_ids": [
177
+ "testB-79644-219272"
178
+ ],
179
+ "batch_size": 1,
180
+ "generation_latency_seconds": 7.480462849023752,
181
+ "batch_wall_latency_seconds": 7.510920200962573,
182
+ "amortized_generation_ms_per_sample": 7480.462849023752,
183
+ "amortized_batch_wall_ms_per_sample": 7510.920200962573,
184
+ "outputs": [
185
+ {
186
+ "raw_response": "{\"objects\": [{\"bbox_2d\":[120,390,270,790],\"score\":0.92},{\"bbox_2d\":[160,430,300,730],\"score\":0.91},{\"bbox_2d\":[270,560,320,740],\"score\":0.90},{\"bbox_2d\":[300,550,350,720],\"score\":0.89},{\"bbox_2d\":[340,540,380,700],\"score\":0.88},{\"bbox_2d\":[450,560,500,770],\"score\":0.87},{\"bbox_2d\":[480,580,530,760],\"score\":0.86},{\"bbox_2d\":[530,560,570,710],\"score\":0.85},{\"bbox_2d\":[570,530,740,780],\"score\":0.84},{\"bbox_2d\":[670,550,740,810],\"score\":0.83},{\"bbox_2d\":[740,540,800,760],\"score\":0.82},{\"bbox_2d\":[800,540,840,700],\"score\":0.81},{\"bbox_2d\":[840,540,900,780],\"score\":0.80}]}",
187
+ "valid_prediction": {
188
+ "objects": [
189
+ {
190
+ "bbox_2d": [
191
+ 120,
192
+ 390,
193
+ 270,
194
+ 790
195
+ ],
196
+ "score": 0.92
197
+ },
198
+ {
199
+ "bbox_2d": [
200
+ 160,
201
+ 430,
202
+ 300,
203
+ 730
204
+ ],
205
+ "score": 0.91
206
+ },
207
+ {
208
+ "bbox_2d": [
209
+ 270,
210
+ 560,
211
+ 320,
212
+ 740
213
+ ],
214
+ "score": 0.9
215
+ },
216
+ {
217
+ "bbox_2d": [
218
+ 300,
219
+ 550,
220
+ 350,
221
+ 720
222
+ ],
223
+ "score": 0.89
224
+ },
225
+ {
226
+ "bbox_2d": [
227
+ 340,
228
+ 540,
229
+ 380,
230
+ 700
231
+ ],
232
+ "score": 0.88
233
+ },
234
+ {
235
+ "bbox_2d": [
236
+ 450,
237
+ 560,
238
+ 500,
239
+ 770
240
+ ],
241
+ "score": 0.87
242
+ },
243
+ {
244
+ "bbox_2d": [
245
+ 480,
246
+ 580,
247
+ 530,
248
+ 760
249
+ ],
250
+ "score": 0.86
251
+ },
252
+ {
253
+ "bbox_2d": [
254
+ 530,
255
+ 560,
256
+ 570,
257
+ 710
258
+ ],
259
+ "score": 0.85
260
+ },
261
+ {
262
+ "bbox_2d": [
263
+ 570,
264
+ 530,
265
+ 740,
266
+ 780
267
+ ],
268
+ "score": 0.84
269
+ },
270
+ {
271
+ "bbox_2d": [
272
+ 670,
273
+ 550,
274
+ 740,
275
+ 810
276
+ ],
277
+ "score": 0.83
278
+ },
279
+ {
280
+ "bbox_2d": [
281
+ 740,
282
+ 540,
283
+ 800,
284
+ 760
285
+ ],
286
+ "score": 0.82
287
+ },
288
+ {
289
+ "bbox_2d": [
290
+ 800,
291
+ 540,
292
+ 840,
293
+ 700
294
+ ],
295
+ "score": 0.81
296
+ },
297
+ {
298
+ "bbox_2d": [
299
+ 840,
300
+ 540,
301
+ 900,
302
+ 780
303
+ ],
304
+ "score": 0.8
305
+ }
306
+ ]
307
+ },
308
+ "error": null,
309
+ "finish_reason": "stop",
310
+ "completion_tokens": 370,
311
+ "prompt_tokens": 520
312
+ }
313
+ ]
314
+ },
315
+ {
316
+ "sample_ids": [
317
+ "testB-65935-181776"
318
+ ],
319
+ "batch_size": 1,
320
+ "generation_latency_seconds": 10.180860323016532,
321
+ "batch_wall_latency_seconds": 10.195898448000662,
322
+ "amortized_generation_ms_per_sample": 10180.860323016532,
323
+ "amortized_batch_wall_ms_per_sample": 10195.898448000662,
324
+ "outputs": [
325
+ {
326
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[340 ",
327
+ "valid_prediction": null,
328
+ "error": "Expecting ',' delimiter: line 1 column 529 (char 528)",
329
+ "finish_reason": "length",
330
+ "completion_tokens": 512,
331
+ "prompt_tokens": 524
332
+ }
333
+ ]
334
+ },
335
+ {
336
+ "sample_ids": [
337
+ "testA-81622-224664"
338
+ ],
339
+ "batch_size": 1,
340
+ "generation_latency_seconds": 0.2510422539198771,
341
+ "batch_wall_latency_seconds": 0.26205689006019384,
342
+ "amortized_generation_ms_per_sample": 251.0422539198771,
343
+ "amortized_batch_wall_ms_per_sample": 262.05689006019384,
344
+ "outputs": [
345
+ {
346
+ "raw_response": "{\"objects\":[]}",
347
+ "valid_prediction": {
348
+ "objects": []
349
+ },
350
+ "error": null,
351
+ "finish_reason": "stop",
352
+ "completion_tokens": 6,
353
+ "prompt_tokens": 524
354
+ }
355
+ ]
356
+ },
357
+ {
358
+ "sample_ids": [
359
+ "testA-10078-27739"
360
+ ],
361
+ "batch_size": 1,
362
+ "generation_latency_seconds": 0.26944356702733785,
363
+ "batch_wall_latency_seconds": 0.2834722528932616,
364
+ "amortized_generation_ms_per_sample": 269.44356702733785,
365
+ "amortized_batch_wall_ms_per_sample": 283.4722528932616,
366
+ "outputs": [
367
+ {
368
+ "raw_response": "{\"objects\":[]}",
369
+ "valid_prediction": {
370
+ "objects": []
371
+ },
372
+ "error": null,
373
+ "finish_reason": "stop",
374
+ "completion_tokens": 6,
375
+ "prompt_tokens": 525
376
+ }
377
+ ]
378
+ },
379
+ {
380
+ "sample_ids": [
381
+ "testA-15582-42797"
382
+ ],
383
+ "batch_size": 1,
384
+ "generation_latency_seconds": 1.9161002039909363,
385
+ "batch_wall_latency_seconds": 1.9308117219479755,
386
+ "amortized_generation_ms_per_sample": 1916.1002039909363,
387
+ "amortized_batch_wall_ms_per_sample": 1930.8117219479755,
388
+ "outputs": [
389
+ {
390
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[80,280,400,980],\"score\":0.95},{\"bbox_2d\":[310,390,510,980],\"score\":0.93},{\"bbox_2d\":[540,400,640,870],\"score\":0.88}]}",
391
+ "valid_prediction": {
392
+ "objects": [
393
+ {
394
+ "bbox_2d": [
395
+ 80,
396
+ 280,
397
+ 400,
398
+ 980
399
+ ],
400
+ "score": 0.95
401
+ },
402
+ {
403
+ "bbox_2d": [
404
+ 310,
405
+ 390,
406
+ 510,
407
+ 980
408
+ ],
409
+ "score": 0.93
410
+ },
411
+ {
412
+ "bbox_2d": [
413
+ 540,
414
+ 400,
415
+ 640,
416
+ 870
417
+ ],
418
+ "score": 0.88
419
+ }
420
+ ]
421
+ },
422
+ "error": null,
423
+ "finish_reason": "stop",
424
+ "completion_tokens": 88,
425
+ "prompt_tokens": 539
426
+ }
427
+ ]
428
+ },
429
+ {
430
+ "sample_ids": [
431
+ "testA-5038-13943"
432
+ ],
433
+ "batch_size": 1,
434
+ "generation_latency_seconds": 0.7828407019842416,
435
+ "batch_wall_latency_seconds": 0.7960362600861117,
436
+ "amortized_generation_ms_per_sample": 782.8407019842416,
437
+ "amortized_batch_wall_ms_per_sample": 796.0362600861117,
438
+ "outputs": [
439
+ {
440
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[490,240,780,980],\"score\":0.95}]}",
441
+ "valid_prediction": {
442
+ "objects": [
443
+ {
444
+ "bbox_2d": [
445
+ 490,
446
+ 240,
447
+ 780,
448
+ 980
449
+ ],
450
+ "score": 0.95
451
+ }
452
+ ]
453
+ },
454
+ "error": null,
455
+ "finish_reason": "stop",
456
+ "completion_tokens": 33,
457
+ "prompt_tokens": 524
458
+ }
459
+ ]
460
+ },
461
+ {
462
+ "sample_ids": [
463
+ "testA-30844-84861"
464
+ ],
465
+ "batch_size": 1,
466
+ "generation_latency_seconds": 10.480434139957651,
467
+ "batch_wall_latency_seconds": 10.485813157982193,
468
+ "amortized_generation_ms_per_sample": 10480.434139957651,
469
+ "amortized_batch_wall_ms_per_sample": 10485.813157982193,
470
+ "outputs": [
471
+ {
472
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[470 \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n",
473
+ "valid_prediction": null,
474
+ "error": "Expecting ',' delimiter: line 499 column 1 (char 2020)",
475
+ "finish_reason": "length",
476
+ "completion_tokens": 512,
477
+ "prompt_tokens": 543
478
+ }
479
+ ]
480
+ },
481
+ {
482
+ "sample_ids": [
483
+ "testB-15419-42356"
484
+ ],
485
+ "batch_size": 1,
486
+ "generation_latency_seconds": 10.714943284052424,
487
+ "batch_wall_latency_seconds": 10.730338808032684,
488
+ "amortized_generation_ms_per_sample": 10714.943284052424,
489
+ "amortized_batch_wall_ms_per_sample": 10730.338808032684,
490
+ "outputs": [
491
+ {
492
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[110 ",
493
+ "valid_prediction": null,
494
+ "error": "Expecting ',' delimiter: line 1 column 529 (char 528)",
495
+ "finish_reason": "length",
496
+ "completion_tokens": 512,
497
+ "prompt_tokens": 526
498
+ }
499
+ ]
500
+ },
501
+ {
502
+ "sample_ids": [
503
+ "testB-27486-75580"
504
+ ],
505
+ "batch_size": 1,
506
+ "generation_latency_seconds": 10.180824494105764,
507
+ "batch_wall_latency_seconds": 10.188868912984617,
508
+ "amortized_generation_ms_per_sample": 10180.824494105764,
509
+ "amortized_batch_wall_ms_per_sample": 10188.868912984617,
510
+ "outputs": [
511
+ {
512
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[530 \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n",
513
+ "valid_prediction": null,
514
+ "error": "Expecting ',' delimiter: line 499 column 1 (char 2020)",
515
+ "finish_reason": "length",
516
+ "completion_tokens": 512,
517
+ "prompt_tokens": 530
518
+ }
519
+ ]
520
+ },
521
+ {
522
+ "sample_ids": [
523
+ "testB-945-2632"
524
+ ],
525
+ "batch_size": 1,
526
+ "generation_latency_seconds": 3.843070151982829,
527
+ "batch_wall_latency_seconds": 3.859969821991399,
528
+ "amortized_generation_ms_per_sample": 3843.070151982829,
529
+ "amortized_batch_wall_ms_per_sample": 3859.969821991399,
530
+ "outputs": [
531
+ {
532
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[120,530,220,910] ,\"score\":0.92},{\"bbox_2d\":[250,530,350,910] ,\"score\":0.92},{\"bbox_2d\":[380,530,480,910] ,\"score\":0.92},{\"bbox_2d\":[500,520,600,910] ,\"score\":0.92},{\"bbox_2d\":[620,530,720,910] ,\"score\":0.92},{\"bbox_2d\":[740,530,840,910] ,\"score\":0.92}]}",
533
+ "valid_prediction": {
534
+ "objects": [
535
+ {
536
+ "bbox_2d": [
537
+ 120,
538
+ 530,
539
+ 220,
540
+ 910
541
+ ],
542
+ "score": 0.92
543
+ },
544
+ {
545
+ "bbox_2d": [
546
+ 250,
547
+ 530,
548
+ 350,
549
+ 910
550
+ ],
551
+ "score": 0.92
552
+ },
553
+ {
554
+ "bbox_2d": [
555
+ 380,
556
+ 530,
557
+ 480,
558
+ 910
559
+ ],
560
+ "score": 0.92
561
+ },
562
+ {
563
+ "bbox_2d": [
564
+ 500,
565
+ 520,
566
+ 600,
567
+ 910
568
+ ],
569
+ "score": 0.92
570
+ },
571
+ {
572
+ "bbox_2d": [
573
+ 620,
574
+ 530,
575
+ 720,
576
+ 910
577
+ ],
578
+ "score": 0.92
579
+ },
580
+ {
581
+ "bbox_2d": [
582
+ 740,
583
+ 530,
584
+ 840,
585
+ 910
586
+ ],
587
+ "score": 0.92
588
+ }
589
+ ]
590
+ },
591
+ "error": null,
592
+ "finish_reason": "stop",
593
+ "completion_tokens": 179,
594
+ "prompt_tokens": 530
595
+ }
596
+ ]
597
+ },
598
+ {
599
+ "sample_ids": [
600
+ "testA-24438-67190"
601
+ ],
602
+ "batch_size": 1,
603
+ "generation_latency_seconds": 10.143939319998026,
604
+ "batch_wall_latency_seconds": 10.162826716084965,
605
+ "amortized_generation_ms_per_sample": 10143.939319998026,
606
+ "amortized_batch_wall_ms_per_sample": 10162.826716084965,
607
+ "outputs": [
608
+ {
609
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[290,240,590,920],\"score\":0.95},{\"bbox_2d\":[730 \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n",
610
+ "valid_prediction": null,
611
+ "error": "Expecting ',' delimiter: line 472 column 1 (char 1954)",
612
+ "finish_reason": "length",
613
+ "completion_tokens": 512,
614
+ "prompt_tokens": 525
615
+ }
616
+ ]
617
+ },
618
+ {
619
+ "sample_ids": [
620
+ "testB-66161-182405"
621
+ ],
622
+ "batch_size": 1,
623
+ "generation_latency_seconds": 10.25821736396756,
624
+ "batch_wall_latency_seconds": 10.271643411950208,
625
+ "amortized_generation_ms_per_sample": 10258.21736396756,
626
+ "amortized_batch_wall_ms_per_sample": 10271.643411950208,
627
+ "outputs": [
628
+ {
629
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[100 ",
630
+ "valid_prediction": null,
631
+ "error": "Expecting ',' delimiter: line 1 column 529 (char 528)",
632
+ "finish_reason": "length",
633
+ "completion_tokens": 512,
634
+ "prompt_tokens": 523
635
+ }
636
+ ]
637
+ },
638
+ {
639
+ "sample_ids": [
640
+ "testB-42726-117659"
641
+ ],
642
+ "batch_size": 1,
643
+ "generation_latency_seconds": 0.9900629689218476,
644
+ "batch_wall_latency_seconds": 1.0045652500120923,
645
+ "amortized_generation_ms_per_sample": 990.0629689218476,
646
+ "amortized_batch_wall_ms_per_sample": 1004.5652500120923,
647
+ "outputs": [
648
+ {
649
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[775 ,775 ,999 ,999 ],\"score\":0.92}]}",
650
+ "valid_prediction": {
651
+ "objects": [
652
+ {
653
+ "bbox_2d": [
654
+ 775,
655
+ 775,
656
+ 999,
657
+ 999
658
+ ],
659
+ "score": 0.92
660
+ }
661
+ ]
662
+ },
663
+ "error": null,
664
+ "finish_reason": "stop",
665
+ "completion_tokens": 43,
666
+ "prompt_tokens": 525
667
+ }
668
+ ]
669
+ },
670
+ {
671
+ "sample_ids": [
672
+ "testB-69439-191450"
673
+ ],
674
+ "batch_size": 1,
675
+ "generation_latency_seconds": 0.19075044698547572,
676
+ "batch_wall_latency_seconds": 0.20009391102939844,
677
+ "amortized_generation_ms_per_sample": 190.75044698547572,
678
+ "amortized_batch_wall_ms_per_sample": 200.09391102939844,
679
+ "outputs": [
680
+ {
681
+ "raw_response": "{\"objects\":[]}",
682
+ "valid_prediction": {
683
+ "objects": []
684
+ },
685
+ "error": null,
686
+ "finish_reason": "stop",
687
+ "completion_tokens": 5,
688
+ "prompt_tokens": 388
689
+ }
690
+ ]
691
+ },
692
+ {
693
+ "sample_ids": [
694
+ "testA-54307-149426"
695
+ ],
696
+ "batch_size": 1,
697
+ "generation_latency_seconds": 1.2734235550742596,
698
+ "batch_wall_latency_seconds": 1.2819705069996417,
699
+ "amortized_generation_ms_per_sample": 1273.4235550742596,
700
+ "amortized_batch_wall_ms_per_sample": 1281.9705069996417,
701
+ "outputs": [
702
+ {
703
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[170,0,560,420] ,\"score\":0.95},{\"bbox_2d\":[540,40,830,910] ,\"score\":0.96}]}",
704
+ "valid_prediction": {
705
+ "objects": [
706
+ {
707
+ "bbox_2d": [
708
+ 170,
709
+ 0,
710
+ 560,
711
+ 420
712
+ ],
713
+ "score": 0.95
714
+ },
715
+ {
716
+ "bbox_2d": [
717
+ 540,
718
+ 40,
719
+ 830,
720
+ 910
721
+ ],
722
+ "score": 0.96
723
+ }
724
+ ]
725
+ },
726
+ "error": null,
727
+ "finish_reason": "stop",
728
+ "completion_tokens": 60,
729
+ "prompt_tokens": 524
730
+ }
731
+ ]
732
+ },
733
+ {
734
+ "sample_ids": [
735
+ "testB-37669-103831"
736
+ ],
737
+ "batch_size": 1,
738
+ "generation_latency_seconds": 0.21837474894709885,
739
+ "batch_wall_latency_seconds": 0.23633889202028513,
740
+ "amortized_generation_ms_per_sample": 218.37474894709885,
741
+ "amortized_batch_wall_ms_per_sample": 236.33889202028513,
742
+ "outputs": [
743
+ {
744
+ "raw_response": "{\"objects\":[]}",
745
+ "valid_prediction": {
746
+ "objects": []
747
+ },
748
+ "error": null,
749
+ "finish_reason": "stop",
750
+ "completion_tokens": 5,
751
+ "prompt_tokens": 528
752
+ }
753
+ ]
754
+ },
755
+ {
756
+ "sample_ids": [
757
+ "testB-50844-140105"
758
+ ],
759
+ "batch_size": 1,
760
+ "generation_latency_seconds": 0.23954864998813719,
761
+ "batch_wall_latency_seconds": 0.2525435209972784,
762
+ "amortized_generation_ms_per_sample": 239.54864998813719,
763
+ "amortized_batch_wall_ms_per_sample": 252.5435209972784,
764
+ "outputs": [
765
+ {
766
+ "raw_response": "{\"objects\":[]}",
767
+ "valid_prediction": {
768
+ "objects": []
769
+ },
770
+ "error": null,
771
+ "finish_reason": "stop",
772
+ "completion_tokens": 6,
773
+ "prompt_tokens": 522
774
+ }
775
+ ]
776
+ },
777
+ {
778
+ "sample_ids": [
779
+ "testA-79438-218690"
780
+ ],
781
+ "batch_size": 1,
782
+ "generation_latency_seconds": 0.7543689489830285,
783
+ "batch_wall_latency_seconds": 0.7701102939900011,
784
+ "amortized_generation_ms_per_sample": 754.3689489830285,
785
+ "amortized_batch_wall_ms_per_sample": 770.1102939900011,
786
+ "outputs": [
787
+ {
788
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[260,170,520,910],\"score\":0.95}]}",
789
+ "valid_prediction": {
790
+ "objects": [
791
+ {
792
+ "bbox_2d": [
793
+ 260,
794
+ 170,
795
+ 520,
796
+ 910
797
+ ],
798
+ "score": 0.95
799
+ }
800
+ ]
801
+ },
802
+ "error": null,
803
+ "finish_reason": "stop",
804
+ "completion_tokens": 33,
805
+ "prompt_tokens": 522
806
+ }
807
+ ]
808
+ },
809
+ {
810
+ "sample_ids": [
811
+ "testA-62940-173454"
812
+ ],
813
+ "batch_size": 1,
814
+ "generation_latency_seconds": 0.8060770370066166,
815
+ "batch_wall_latency_seconds": 0.8215904459357262,
816
+ "amortized_generation_ms_per_sample": 806.0770370066166,
817
+ "amortized_batch_wall_ms_per_sample": 821.5904459357262,
818
+ "outputs": [
819
+ {
820
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[620,270,750,690],\"score\":0.95}]}",
821
+ "valid_prediction": {
822
+ "objects": [
823
+ {
824
+ "bbox_2d": [
825
+ 620,
826
+ 270,
827
+ 750,
828
+ 690
829
+ ],
830
+ "score": 0.95
831
+ }
832
+ ]
833
+ },
834
+ "error": null,
835
+ "finish_reason": "stop",
836
+ "completion_tokens": 33,
837
+ "prompt_tokens": 525
838
+ }
839
+ ]
840
+ },
841
+ {
842
+ "sample_ids": [
843
+ "testA-81245-223632"
844
+ ],
845
+ "batch_size": 1,
846
+ "generation_latency_seconds": 2.46771560003981,
847
+ "batch_wall_latency_seconds": 2.4794425220461562,
848
+ "amortized_generation_ms_per_sample": 2467.71560003981,
849
+ "amortized_batch_wall_ms_per_sample": 2479.4425220461562,
850
+ "outputs": [
851
+ {
852
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[530,110,760,880] ,\"score\":0.95},{\"bbox_2d\":[560,180,630,610] ,\"score\":0.93},{\"bbox_2d\":[690,130,790,740] ,\"score\":0.91},{\"bbox_2d\":[790,110,920,660] ,\"score\":0.90}]}",
853
+ "valid_prediction": {
854
+ "objects": [
855
+ {
856
+ "bbox_2d": [
857
+ 530,
858
+ 110,
859
+ 760,
860
+ 880
861
+ ],
862
+ "score": 0.95
863
+ },
864
+ {
865
+ "bbox_2d": [
866
+ 560,
867
+ 180,
868
+ 630,
869
+ 610
870
+ ],
871
+ "score": 0.93
872
+ },
873
+ {
874
+ "bbox_2d": [
875
+ 690,
876
+ 130,
877
+ 790,
878
+ 740
879
+ ],
880
+ "score": 0.91
881
+ },
882
+ {
883
+ "bbox_2d": [
884
+ 790,
885
+ 110,
886
+ 920,
887
+ 660
888
+ ],
889
+ "score": 0.9
890
+ }
891
+ ]
892
+ },
893
+ "error": null,
894
+ "finish_reason": "stop",
895
+ "completion_tokens": 121,
896
+ "prompt_tokens": 530
897
+ }
898
+ ]
899
+ },
900
+ {
901
+ "sample_ids": [
902
+ "testA-47070-129599"
903
+ ],
904
+ "batch_size": 1,
905
+ "generation_latency_seconds": 0.22282294591423124,
906
+ "batch_wall_latency_seconds": 0.2299416270107031,
907
+ "amortized_generation_ms_per_sample": 222.82294591423124,
908
+ "amortized_batch_wall_ms_per_sample": 229.9416270107031,
909
+ "outputs": [
910
+ {
911
+ "raw_response": "{\"objects\":[]}",
912
+ "valid_prediction": {
913
+ "objects": []
914
+ },
915
+ "error": null,
916
+ "finish_reason": "stop",
917
+ "completion_tokens": 5,
918
+ "prompt_tokens": 521
919
+ }
920
+ ]
921
+ },
922
+ {
923
+ "sample_ids": [
924
+ "testB-12381-34098"
925
+ ],
926
+ "batch_size": 1,
927
+ "generation_latency_seconds": 10.451599241001531,
928
+ "batch_wall_latency_seconds": 10.457325526047498,
929
+ "amortized_generation_ms_per_sample": 10451.599241001531,
930
+ "amortized_batch_wall_ms_per_sample": 10457.325526047498,
931
+ "outputs": [
932
+ {
933
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[290 ",
934
+ "valid_prediction": null,
935
+ "error": "Expecting ',' delimiter: line 1 column 529 (char 528)",
936
+ "finish_reason": "length",
937
+ "completion_tokens": 512,
938
+ "prompt_tokens": 534
939
+ }
940
+ ]
941
+ },
942
+ {
943
+ "sample_ids": [
944
+ "testA-44142-121498"
945
+ ],
946
+ "batch_size": 1,
947
+ "generation_latency_seconds": 1.4406404530163854,
948
+ "batch_wall_latency_seconds": 1.4515395000344142,
949
+ "amortized_generation_ms_per_sample": 1440.6404530163854,
950
+ "amortized_batch_wall_ms_per_sample": 1451.5395000344142,
951
+ "outputs": [
952
+ {
953
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[0,100,580,980] ,\"score\":0.95},{\"bbox_2d\":[570,120,1000,900] ,\"score\":0.93}]}",
954
+ "valid_prediction": {
955
+ "objects": [
956
+ {
957
+ "bbox_2d": [
958
+ 0,
959
+ 100,
960
+ 580,
961
+ 980
962
+ ],
963
+ "score": 0.95
964
+ },
965
+ {
966
+ "bbox_2d": [
967
+ 570,
968
+ 120,
969
+ 1000,
970
+ 900
971
+ ],
972
+ "score": 0.93
973
+ }
974
+ ]
975
+ },
976
+ "error": null,
977
+ "finish_reason": "stop",
978
+ "completion_tokens": 62,
979
+ "prompt_tokens": 523
980
+ }
981
+ ]
982
+ },
983
+ {
984
+ "sample_ids": [
985
+ "testA-73258-201893"
986
+ ],
987
+ "batch_size": 1,
988
+ "generation_latency_seconds": 11.113033535890281,
989
+ "batch_wall_latency_seconds": 11.123388323001564,
990
+ "amortized_generation_ms_per_sample": 11113.033535890281,
991
+ "amortized_batch_wall_ms_per_sample": 11123.388323001564,
992
+ "outputs": [
993
+ {
994
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[520 \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n",
995
+ "valid_prediction": null,
996
+ "error": "Expecting ',' delimiter: line 251 column 1 (char 1401)",
997
+ "finish_reason": "length",
998
+ "completion_tokens": 512,
999
+ "prompt_tokens": 526
1000
+ }
1001
+ ]
1002
+ },
1003
+ {
1004
+ "sample_ids": [
1005
+ "testB-46644-128436"
1006
+ ],
1007
+ "batch_size": 1,
1008
+ "generation_latency_seconds": 1.2878015459282324,
1009
+ "batch_wall_latency_seconds": 1.2976085640257224,
1010
+ "amortized_generation_ms_per_sample": 1287.8015459282324,
1011
+ "amortized_batch_wall_ms_per_sample": 1297.6085640257224,
1012
+ "outputs": [
1013
+ {
1014
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[100,180,520,720],\"score\":0.95},{\"bbox_2d\":[450,0,990,740],\"score\":0.96}]}",
1015
+ "valid_prediction": {
1016
+ "objects": [
1017
+ {
1018
+ "bbox_2d": [
1019
+ 100,
1020
+ 180,
1021
+ 520,
1022
+ 720
1023
+ ],
1024
+ "score": 0.95
1025
+ },
1026
+ {
1027
+ "bbox_2d": [
1028
+ 450,
1029
+ 0,
1030
+ 990,
1031
+ 740
1032
+ ],
1033
+ "score": 0.96
1034
+ }
1035
+ ]
1036
+ },
1037
+ "error": null,
1038
+ "finish_reason": "stop",
1039
+ "completion_tokens": 59,
1040
+ "prompt_tokens": 393
1041
+ }
1042
+ ]
1043
+ },
1044
+ {
1045
+ "sample_ids": [
1046
+ "testA-43889-120792"
1047
+ ],
1048
+ "batch_size": 1,
1049
+ "generation_latency_seconds": 0.7570710440631956,
1050
+ "batch_wall_latency_seconds": 0.7649242469342425,
1051
+ "amortized_generation_ms_per_sample": 757.0710440631956,
1052
+ "amortized_batch_wall_ms_per_sample": 764.9242469342425,
1053
+ "outputs": [
1054
+ {
1055
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[120,260,340,830],\"score\":0.95}]}",
1056
+ "valid_prediction": {
1057
+ "objects": [
1058
+ {
1059
+ "bbox_2d": [
1060
+ 120,
1061
+ 260,
1062
+ 340,
1063
+ 830
1064
+ ],
1065
+ "score": 0.95
1066
+ }
1067
+ ]
1068
+ },
1069
+ "error": null,
1070
+ "finish_reason": "stop",
1071
+ "completion_tokens": 33,
1072
+ "prompt_tokens": 531
1073
+ }
1074
+ ]
1075
+ },
1076
+ {
1077
+ "sample_ids": [
1078
+ "testB-36082-99363"
1079
+ ],
1080
+ "batch_size": 1,
1081
+ "generation_latency_seconds": 1.2985183690907434,
1082
+ "batch_wall_latency_seconds": 1.3064349760534242,
1083
+ "amortized_generation_ms_per_sample": 1298.5183690907434,
1084
+ "amortized_batch_wall_ms_per_sample": 1306.4349760534242,
1085
+ "outputs": [
1086
+ {
1087
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[0,100,420,890] ,\"score\":0.95},{\"bbox_2d\":[440,100,990,890] ,\"score\":0.95}]}",
1088
+ "valid_prediction": {
1089
+ "objects": [
1090
+ {
1091
+ "bbox_2d": [
1092
+ 0,
1093
+ 100,
1094
+ 420,
1095
+ 890
1096
+ ],
1097
+ "score": 0.95
1098
+ },
1099
+ {
1100
+ "bbox_2d": [
1101
+ 440,
1102
+ 100,
1103
+ 990,
1104
+ 890
1105
+ ],
1106
+ "score": 0.95
1107
+ }
1108
+ ]
1109
+ },
1110
+ "error": null,
1111
+ "finish_reason": "stop",
1112
+ "completion_tokens": 61,
1113
+ "prompt_tokens": 392
1114
+ }
1115
+ ]
1116
+ },
1117
+ {
1118
+ "sample_ids": [
1119
+ "testB-33108-91152"
1120
+ ],
1121
+ "batch_size": 1,
1122
+ "generation_latency_seconds": 0.9395933509804308,
1123
+ "batch_wall_latency_seconds": 0.9509444029536098,
1124
+ "amortized_generation_ms_per_sample": 939.5933509804308,
1125
+ "amortized_batch_wall_ms_per_sample": 950.9444029536098,
1126
+ "outputs": [
1127
+ {
1128
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[0 ,270 ,390 ,450 ],\"score\":0.95}]}",
1129
+ "valid_prediction": {
1130
+ "objects": [
1131
+ {
1132
+ "bbox_2d": [
1133
+ 0,
1134
+ 270,
1135
+ 390,
1136
+ 450
1137
+ ],
1138
+ "score": 0.95
1139
+ }
1140
+ ]
1141
+ },
1142
+ "error": null,
1143
+ "finish_reason": "stop",
1144
+ "completion_tokens": 41,
1145
+ "prompt_tokens": 522
1146
+ }
1147
+ ]
1148
+ },
1149
+ {
1150
+ "sample_ids": [
1151
+ "testA-26856-73830"
1152
+ ],
1153
+ "batch_size": 1,
1154
+ "generation_latency_seconds": 10.264511390007101,
1155
+ "batch_wall_latency_seconds": 10.279962090076879,
1156
+ "amortized_generation_ms_per_sample": 10264.511390007101,
1157
+ "amortized_batch_wall_ms_per_sample": 10279.962090076879,
1158
+ "outputs": [
1159
+ {
1160
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[575 ",
1161
+ "valid_prediction": null,
1162
+ "error": "Expecting ',' delimiter: line 1 column 529 (char 528)",
1163
+ "finish_reason": "length",
1164
+ "completion_tokens": 512,
1165
+ "prompt_tokens": 522
1166
+ }
1167
+ ]
1168
+ },
1169
+ {
1170
+ "sample_ids": [
1171
+ "testA-81271-223709"
1172
+ ],
1173
+ "batch_size": 1,
1174
+ "generation_latency_seconds": 0.23597171693108976,
1175
+ "batch_wall_latency_seconds": 0.2577737799147144,
1176
+ "amortized_generation_ms_per_sample": 235.97171693108976,
1177
+ "amortized_batch_wall_ms_per_sample": 257.7737799147144,
1178
+ "outputs": [
1179
+ {
1180
+ "raw_response": "{\"objects\":[]}",
1181
+ "valid_prediction": {
1182
+ "objects": []
1183
+ },
1184
+ "error": null,
1185
+ "finish_reason": "stop",
1186
+ "completion_tokens": 6,
1187
+ "prompt_tokens": 520
1188
+ }
1189
+ ]
1190
+ },
1191
+ {
1192
+ "sample_ids": [
1193
+ "testA-54554-150141"
1194
+ ],
1195
+ "batch_size": 1,
1196
+ "generation_latency_seconds": 0.6926666980143636,
1197
+ "batch_wall_latency_seconds": 0.710583986947313,
1198
+ "amortized_generation_ms_per_sample": 692.6666980143636,
1199
+ "amortized_batch_wall_ms_per_sample": 710.583986947313,
1200
+ "outputs": [
1201
+ {
1202
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[0,0,1000,250] ,\"score\":0.95}]}",
1203
+ "valid_prediction": {
1204
+ "objects": [
1205
+ {
1206
+ "bbox_2d": [
1207
+ 0,
1208
+ 0,
1209
+ 1000,
1210
+ 250
1211
+ ],
1212
+ "score": 0.95
1213
+ }
1214
+ ]
1215
+ },
1216
+ "error": null,
1217
+ "finish_reason": "stop",
1218
+ "completion_tokens": 31,
1219
+ "prompt_tokens": 530
1220
+ }
1221
+ ]
1222
+ },
1223
+ {
1224
+ "sample_ids": [
1225
+ "testB-38636-106481"
1226
+ ],
1227
+ "batch_size": 1,
1228
+ "generation_latency_seconds": 9.935319383977912,
1229
+ "batch_wall_latency_seconds": 9.945194306899793,
1230
+ "amortized_generation_ms_per_sample": 9935.319383977912,
1231
+ "amortized_batch_wall_ms_per_sample": 9945.194306899793,
1232
+ "outputs": [
1233
+ {
1234
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[430 ",
1235
+ "valid_prediction": null,
1236
+ "error": "Expecting ',' delimiter: line 1 column 529 (char 528)",
1237
+ "finish_reason": "length",
1238
+ "completion_tokens": 512,
1239
+ "prompt_tokens": 397
1240
+ }
1241
+ ]
1242
+ },
1243
+ {
1244
+ "sample_ids": [
1245
+ "testA-5734-15837"
1246
+ ],
1247
+ "batch_size": 1,
1248
+ "generation_latency_seconds": 1.2923860140144825,
1249
+ "batch_wall_latency_seconds": 1.3085041570011526,
1250
+ "amortized_generation_ms_per_sample": 1292.3860140144825,
1251
+ "amortized_batch_wall_ms_per_sample": 1308.5041570011526,
1252
+ "outputs": [
1253
+ {
1254
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[0,190,310,740],\"score\":0.95},{\"bbox_2d\":[400,230,660,690],\"score\":0.93}]}",
1255
+ "valid_prediction": {
1256
+ "objects": [
1257
+ {
1258
+ "bbox_2d": [
1259
+ 0,
1260
+ 190,
1261
+ 310,
1262
+ 740
1263
+ ],
1264
+ "score": 0.95
1265
+ },
1266
+ {
1267
+ "bbox_2d": [
1268
+ 400,
1269
+ 230,
1270
+ 660,
1271
+ 690
1272
+ ],
1273
+ "score": 0.93
1274
+ }
1275
+ ]
1276
+ },
1277
+ "error": null,
1278
+ "finish_reason": "stop",
1279
+ "completion_tokens": 59,
1280
+ "prompt_tokens": 525
1281
+ }
1282
+ ]
1283
+ },
1284
+ {
1285
+ "sample_ids": [
1286
+ "testB-67558-186325"
1287
+ ],
1288
+ "batch_size": 1,
1289
+ "generation_latency_seconds": 0.23707255406770855,
1290
+ "batch_wall_latency_seconds": 0.24415827100165188,
1291
+ "amortized_generation_ms_per_sample": 237.07255406770855,
1292
+ "amortized_batch_wall_ms_per_sample": 244.15827100165188,
1293
+ "outputs": [
1294
+ {
1295
+ "raw_response": "{\"objects\":[]}",
1296
+ "valid_prediction": {
1297
+ "objects": []
1298
+ },
1299
+ "error": null,
1300
+ "finish_reason": "stop",
1301
+ "completion_tokens": 5,
1302
+ "prompt_tokens": 520
1303
+ }
1304
+ ]
1305
+ },
1306
+ {
1307
+ "sample_ids": [
1308
+ "testB-47053-129554"
1309
+ ],
1310
+ "batch_size": 1,
1311
+ "generation_latency_seconds": 0.8615183549700305,
1312
+ "batch_wall_latency_seconds": 0.8810114429797977,
1313
+ "amortized_generation_ms_per_sample": 861.5183549700305,
1314
+ "amortized_batch_wall_ms_per_sample": 881.0114429797977,
1315
+ "outputs": [
1316
+ {
1317
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[420 ,300 ,800 ,800 ],\"score\":0.95}]}",
1318
+ "valid_prediction": {
1319
+ "objects": [
1320
+ {
1321
+ "bbox_2d": [
1322
+ 420,
1323
+ 300,
1324
+ 800,
1325
+ 800
1326
+ ],
1327
+ "score": 0.95
1328
+ }
1329
+ ]
1330
+ },
1331
+ "error": null,
1332
+ "finish_reason": "stop",
1333
+ "completion_tokens": 39,
1334
+ "prompt_tokens": 525
1335
+ }
1336
+ ]
1337
+ },
1338
+ {
1339
+ "sample_ids": [
1340
+ "testA-93064-256422"
1341
+ ],
1342
+ "batch_size": 1,
1343
+ "generation_latency_seconds": 0.7549799080006778,
1344
+ "batch_wall_latency_seconds": 0.7621725639328361,
1345
+ "amortized_generation_ms_per_sample": 754.9799080006778,
1346
+ "amortized_batch_wall_ms_per_sample": 762.172563932836,
1347
+ "outputs": [
1348
+ {
1349
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[100,80,750,980],\"score\":0.95}]}",
1350
+ "valid_prediction": {
1351
+ "objects": [
1352
+ {
1353
+ "bbox_2d": [
1354
+ 100,
1355
+ 80,
1356
+ 750,
1357
+ 980
1358
+ ],
1359
+ "score": 0.95
1360
+ }
1361
+ ]
1362
+ },
1363
+ "error": null,
1364
+ "finish_reason": "stop",
1365
+ "completion_tokens": 32,
1366
+ "prompt_tokens": 530
1367
+ }
1368
+ ]
1369
+ },
1370
+ {
1371
+ "sample_ids": [
1372
+ "testB-60557-166821"
1373
+ ],
1374
+ "batch_size": 1,
1375
+ "generation_latency_seconds": 0.23407667502760887,
1376
+ "batch_wall_latency_seconds": 0.24112907599192113,
1377
+ "amortized_generation_ms_per_sample": 234.07667502760887,
1378
+ "amortized_batch_wall_ms_per_sample": 241.12907599192113,
1379
+ "outputs": [
1380
+ {
1381
+ "raw_response": "{\"objects\":[]}",
1382
+ "valid_prediction": {
1383
+ "objects": []
1384
+ },
1385
+ "error": null,
1386
+ "finish_reason": "stop",
1387
+ "completion_tokens": 5,
1388
+ "prompt_tokens": 520
1389
+ }
1390
+ ]
1391
+ },
1392
+ {
1393
+ "sample_ids": [
1394
+ "testA-55116-151688"
1395
+ ],
1396
+ "batch_size": 1,
1397
+ "generation_latency_seconds": 0.22477965708822012,
1398
+ "batch_wall_latency_seconds": 0.23706394899636507,
1399
+ "amortized_generation_ms_per_sample": 224.77965708822012,
1400
+ "amortized_batch_wall_ms_per_sample": 237.06394899636507,
1401
+ "outputs": [
1402
+ {
1403
+ "raw_response": "{\"objects\":[]}",
1404
+ "valid_prediction": {
1405
+ "objects": []
1406
+ },
1407
+ "error": null,
1408
+ "finish_reason": "stop",
1409
+ "completion_tokens": 5,
1410
+ "prompt_tokens": 520
1411
+ }
1412
+ ]
1413
+ },
1414
+ {
1415
+ "sample_ids": [
1416
+ "testB-65796-181392"
1417
+ ],
1418
+ "batch_size": 1,
1419
+ "generation_latency_seconds": 0.24896689096931368,
1420
+ "batch_wall_latency_seconds": 0.26310552388895303,
1421
+ "amortized_generation_ms_per_sample": 248.96689096931368,
1422
+ "amortized_batch_wall_ms_per_sample": 263.10552388895303,
1423
+ "outputs": [
1424
+ {
1425
+ "raw_response": "{\"objects\":[]}",
1426
+ "valid_prediction": {
1427
+ "objects": []
1428
+ },
1429
+ "error": null,
1430
+ "finish_reason": "stop",
1431
+ "completion_tokens": 5,
1432
+ "prompt_tokens": 521
1433
+ }
1434
+ ]
1435
+ },
1436
+ {
1437
+ "sample_ids": [
1438
+ "testB-23014-63278"
1439
+ ],
1440
+ "batch_size": 1,
1441
+ "generation_latency_seconds": 0.8068696269765496,
1442
+ "batch_wall_latency_seconds": 0.8200291059911251,
1443
+ "amortized_generation_ms_per_sample": 806.8696269765496,
1444
+ "amortized_batch_wall_ms_per_sample": 820.0291059911251,
1445
+ "outputs": [
1446
+ {
1447
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[30,300,340,700],\"score\":0.95}]}",
1448
+ "valid_prediction": {
1449
+ "objects": [
1450
+ {
1451
+ "bbox_2d": [
1452
+ 30,
1453
+ 300,
1454
+ 340,
1455
+ 700
1456
+ ],
1457
+ "score": 0.95
1458
+ }
1459
+ ]
1460
+ },
1461
+ "error": null,
1462
+ "finish_reason": "stop",
1463
+ "completion_tokens": 32,
1464
+ "prompt_tokens": 523
1465
+ }
1466
+ ]
1467
+ },
1468
+ {
1469
+ "sample_ids": [
1470
+ "testA-24562-67515"
1471
+ ],
1472
+ "batch_size": 1,
1473
+ "generation_latency_seconds": 0.2276058099232614,
1474
+ "batch_wall_latency_seconds": 0.24499452603049576,
1475
+ "amortized_generation_ms_per_sample": 227.6058099232614,
1476
+ "amortized_batch_wall_ms_per_sample": 244.99452603049576,
1477
+ "outputs": [
1478
+ {
1479
+ "raw_response": "{\"objects\":[]}",
1480
+ "valid_prediction": {
1481
+ "objects": []
1482
+ },
1483
+ "error": null,
1484
+ "finish_reason": "stop",
1485
+ "completion_tokens": 5,
1486
+ "prompt_tokens": 526
1487
+ }
1488
+ ]
1489
+ },
1490
+ {
1491
+ "sample_ids": [
1492
+ "testA-17736-48684"
1493
+ ],
1494
+ "batch_size": 1,
1495
+ "generation_latency_seconds": 9.981450814986601,
1496
+ "batch_wall_latency_seconds": 9.995843482087366,
1497
+ "amortized_generation_ms_per_sample": 9981.450814986601,
1498
+ "amortized_batch_wall_ms_per_sample": 9995.843482087366,
1499
+ "outputs": [
1500
+ {
1501
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[500 \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n",
1502
+ "valid_prediction": null,
1503
+ "error": "Expecting ',' delimiter: line 499 column 1 (char 2020)",
1504
+ "finish_reason": "length",
1505
+ "completion_tokens": 512,
1506
+ "prompt_tokens": 525
1507
+ }
1508
+ ]
1509
+ },
1510
+ {
1511
+ "sample_ids": [
1512
+ "testA-58475-160949"
1513
+ ],
1514
+ "batch_size": 1,
1515
+ "generation_latency_seconds": 0.7565385290654376,
1516
+ "batch_wall_latency_seconds": 0.76229489792604,
1517
+ "amortized_generation_ms_per_sample": 756.5385290654376,
1518
+ "amortized_batch_wall_ms_per_sample": 762.29489792604,
1519
+ "outputs": [
1520
+ {
1521
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[40,10,580,980],\"score\":0.95}]}",
1522
+ "valid_prediction": {
1523
+ "objects": [
1524
+ {
1525
+ "bbox_2d": [
1526
+ 40,
1527
+ 10,
1528
+ 580,
1529
+ 980
1530
+ ],
1531
+ "score": 0.95
1532
+ }
1533
+ ]
1534
+ },
1535
+ "error": null,
1536
+ "finish_reason": "stop",
1537
+ "completion_tokens": 31,
1538
+ "prompt_tokens": 525
1539
+ }
1540
+ ]
1541
+ },
1542
+ {
1543
+ "sample_ids": [
1544
+ "testA-80455-221493"
1545
+ ],
1546
+ "batch_size": 1,
1547
+ "generation_latency_seconds": 1.0085041749989614,
1548
+ "batch_wall_latency_seconds": 1.020079139037989,
1549
+ "amortized_generation_ms_per_sample": 1008.5041749989614,
1550
+ "amortized_batch_wall_ms_per_sample": 1020.0791390379891,
1551
+ "outputs": [
1552
+ {
1553
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[0 , 0 , 560 , 890 ],\"score\":0.95}]}",
1554
+ "valid_prediction": {
1555
+ "objects": [
1556
+ {
1557
+ "bbox_2d": [
1558
+ 0,
1559
+ 0,
1560
+ 560,
1561
+ 890
1562
+ ],
1563
+ "score": 0.95
1564
+ }
1565
+ ]
1566
+ },
1567
+ "error": null,
1568
+ "finish_reason": "stop",
1569
+ "completion_tokens": 45,
1570
+ "prompt_tokens": 522
1571
+ }
1572
+ ]
1573
+ },
1574
+ {
1575
+ "sample_ids": [
1576
+ "testA-31763-87405"
1577
+ ],
1578
+ "batch_size": 1,
1579
+ "generation_latency_seconds": 1.3390386949758977,
1580
+ "batch_wall_latency_seconds": 1.3469314540270716,
1581
+ "amortized_generation_ms_per_sample": 1339.0386949758977,
1582
+ "amortized_batch_wall_ms_per_sample": 1346.9314540270716,
1583
+ "outputs": [
1584
+ {
1585
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[350,880,420,930] ,\"score\":0.92},{\"bbox_2d\":[580,750,640,810] ,\"score\":0.91}]}",
1586
+ "valid_prediction": {
1587
+ "objects": [
1588
+ {
1589
+ "bbox_2d": [
1590
+ 350,
1591
+ 880,
1592
+ 420,
1593
+ 930
1594
+ ],
1595
+ "score": 0.92
1596
+ },
1597
+ {
1598
+ "bbox_2d": [
1599
+ 580,
1600
+ 750,
1601
+ 640,
1602
+ 810
1603
+ ],
1604
+ "score": 0.91
1605
+ }
1606
+ ]
1607
+ },
1608
+ "error": null,
1609
+ "finish_reason": "stop",
1610
+ "completion_tokens": 63,
1611
+ "prompt_tokens": 389
1612
+ }
1613
+ ]
1614
+ },
1615
+ {
1616
+ "sample_ids": [
1617
+ "testB-23920-65742"
1618
+ ],
1619
+ "batch_size": 1,
1620
+ "generation_latency_seconds": 10.071104103000835,
1621
+ "batch_wall_latency_seconds": 10.078898616950028,
1622
+ "amortized_generation_ms_per_sample": 10071.104103000835,
1623
+ "amortized_batch_wall_ms_per_sample": 10078.898616950028,
1624
+ "outputs": [
1625
+ {
1626
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[510 \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n",
1627
+ "valid_prediction": null,
1628
+ "error": "Expecting ',' delimiter: line 499 column 1 (char 2020)",
1629
+ "finish_reason": "length",
1630
+ "completion_tokens": 512,
1631
+ "prompt_tokens": 529
1632
+ }
1633
+ ]
1634
+ },
1635
+ {
1636
+ "sample_ids": [
1637
+ "testA-469-1309"
1638
+ ],
1639
+ "batch_size": 1,
1640
+ "generation_latency_seconds": 0.26207074394915253,
1641
+ "batch_wall_latency_seconds": 0.2735602979082614,
1642
+ "amortized_generation_ms_per_sample": 262.07074394915253,
1643
+ "amortized_batch_wall_ms_per_sample": 273.5602979082614,
1644
+ "outputs": [
1645
+ {
1646
+ "raw_response": "{\"objects\":[]}",
1647
+ "valid_prediction": {
1648
+ "objects": []
1649
+ },
1650
+ "error": null,
1651
+ "finish_reason": "stop",
1652
+ "completion_tokens": 5,
1653
+ "prompt_tokens": 522
1654
+ }
1655
+ ]
1656
+ },
1657
+ {
1658
+ "sample_ids": [
1659
+ "testB-56691-155998"
1660
+ ],
1661
+ "batch_size": 1,
1662
+ "generation_latency_seconds": 10.205810929066502,
1663
+ "batch_wall_latency_seconds": 10.223061772994697,
1664
+ "amortized_generation_ms_per_sample": 10205.810929066502,
1665
+ "amortized_batch_wall_ms_per_sample": 10223.061772994697,
1666
+ "outputs": [
1667
+ {
1668
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[690 ",
1669
+ "valid_prediction": null,
1670
+ "error": "Expecting ',' delimiter: line 1 column 529 (char 528)",
1671
+ "finish_reason": "length",
1672
+ "completion_tokens": 512,
1673
+ "prompt_tokens": 523
1674
+ }
1675
+ ]
1676
+ },
1677
+ {
1678
+ "sample_ids": [
1679
+ "testB-31493-86668"
1680
+ ],
1681
+ "batch_size": 1,
1682
+ "generation_latency_seconds": 10.434396203025244,
1683
+ "batch_wall_latency_seconds": 10.448493773932569,
1684
+ "amortized_generation_ms_per_sample": 10434.396203025244,
1685
+ "amortized_batch_wall_ms_per_sample": 10448.493773932569,
1686
+ "outputs": [
1687
+ {
1688
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[158 ",
1689
+ "valid_prediction": null,
1690
+ "error": "Expecting ',' delimiter: line 1 column 529 (char 528)",
1691
+ "finish_reason": "length",
1692
+ "completion_tokens": 512,
1693
+ "prompt_tokens": 535
1694
+ }
1695
+ ]
1696
+ },
1697
+ {
1698
+ "sample_ids": [
1699
+ "testA-59284-163254"
1700
+ ],
1701
+ "batch_size": 1,
1702
+ "generation_latency_seconds": 1.32271799701266,
1703
+ "batch_wall_latency_seconds": 1.3361050819512457,
1704
+ "amortized_generation_ms_per_sample": 1322.71799701266,
1705
+ "amortized_batch_wall_ms_per_sample": 1336.1050819512457,
1706
+ "outputs": [
1707
+ {
1708
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[490,360,680,970],\"score\":0.95},{\"bbox_2d\":[580,310,780,370],\"score\":0.92}]}",
1709
+ "valid_prediction": {
1710
+ "objects": [
1711
+ {
1712
+ "bbox_2d": [
1713
+ 490,
1714
+ 360,
1715
+ 680,
1716
+ 970
1717
+ ],
1718
+ "score": 0.95
1719
+ },
1720
+ {
1721
+ "bbox_2d": [
1722
+ 580,
1723
+ 310,
1724
+ 780,
1725
+ 370
1726
+ ],
1727
+ "score": 0.92
1728
+ }
1729
+ ]
1730
+ },
1731
+ "error": null,
1732
+ "finish_reason": "stop",
1733
+ "completion_tokens": 61,
1734
+ "prompt_tokens": 543
1735
+ }
1736
+ ]
1737
+ },
1738
+ {
1739
+ "sample_ids": [
1740
+ "testA-57397-158015"
1741
+ ],
1742
+ "batch_size": 1,
1743
+ "generation_latency_seconds": 0.22141807794105262,
1744
+ "batch_wall_latency_seconds": 0.2377260560169816,
1745
+ "amortized_generation_ms_per_sample": 221.41807794105262,
1746
+ "amortized_batch_wall_ms_per_sample": 237.7260560169816,
1747
+ "outputs": [
1748
+ {
1749
+ "raw_response": "{\"objects\":[]}",
1750
+ "valid_prediction": {
1751
+ "objects": []
1752
+ },
1753
+ "error": null,
1754
+ "finish_reason": "stop",
1755
+ "completion_tokens": 5,
1756
+ "prompt_tokens": 522
1757
+ }
1758
+ ]
1759
+ },
1760
+ {
1761
+ "sample_ids": [
1762
+ "testB-2935-8111"
1763
+ ],
1764
+ "batch_size": 1,
1765
+ "generation_latency_seconds": 10.570877194055356,
1766
+ "batch_wall_latency_seconds": 10.580568464938551,
1767
+ "amortized_generation_ms_per_sample": 10570.877194055356,
1768
+ "amortized_batch_wall_ms_per_sample": 10580.568464938551,
1769
+ "outputs": [
1770
+ {
1771
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[380 ",
1772
+ "valid_prediction": null,
1773
+ "error": "Expecting ',' delimiter: line 1 column 529 (char 528)",
1774
+ "finish_reason": "length",
1775
+ "completion_tokens": 512,
1776
+ "prompt_tokens": 530
1777
+ }
1778
+ ]
1779
+ },
1780
+ {
1781
+ "sample_ids": [
1782
+ "testB-22290-61226"
1783
+ ],
1784
+ "batch_size": 1,
1785
+ "generation_latency_seconds": 0.22713651705998927,
1786
+ "batch_wall_latency_seconds": 0.23939726105891168,
1787
+ "amortized_generation_ms_per_sample": 227.13651705998927,
1788
+ "amortized_batch_wall_ms_per_sample": 239.39726105891168,
1789
+ "outputs": [
1790
+ {
1791
+ "raw_response": "{\"objects\":[]}",
1792
+ "valid_prediction": {
1793
+ "objects": []
1794
+ },
1795
+ "error": null,
1796
+ "finish_reason": "stop",
1797
+ "completion_tokens": 5,
1798
+ "prompt_tokens": 526
1799
+ }
1800
+ ]
1801
+ },
1802
+ {
1803
+ "sample_ids": [
1804
+ "testB-8416-23140"
1805
+ ],
1806
+ "batch_size": 1,
1807
+ "generation_latency_seconds": 1.8887132819509134,
1808
+ "batch_wall_latency_seconds": 1.9046463000122458,
1809
+ "amortized_generation_ms_per_sample": 1888.7132819509134,
1810
+ "amortized_batch_wall_ms_per_sample": 1904.6463000122458,
1811
+ "outputs": [
1812
+ {
1813
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[0,280,330,720] ,\"score\":0.95},{\"bbox_2d\":[170,270,560,720] ,\"score\":0.94},{\"bbox_2d\":[730,190,960,790] ,\"score\":0.93}]}",
1814
+ "valid_prediction": {
1815
+ "objects": [
1816
+ {
1817
+ "bbox_2d": [
1818
+ 0,
1819
+ 280,
1820
+ 330,
1821
+ 720
1822
+ ],
1823
+ "score": 0.95
1824
+ },
1825
+ {
1826
+ "bbox_2d": [
1827
+ 170,
1828
+ 270,
1829
+ 560,
1830
+ 720
1831
+ ],
1832
+ "score": 0.94
1833
+ },
1834
+ {
1835
+ "bbox_2d": [
1836
+ 730,
1837
+ 190,
1838
+ 960,
1839
+ 790
1840
+ ],
1841
+ "score": 0.93
1842
+ }
1843
+ ]
1844
+ },
1845
+ "error": null,
1846
+ "finish_reason": "stop",
1847
+ "completion_tokens": 90,
1848
+ "prompt_tokens": 525
1849
+ }
1850
+ ]
1851
+ },
1852
+ {
1853
+ "sample_ids": [
1854
+ "testA-60731-167320"
1855
+ ],
1856
+ "batch_size": 1,
1857
+ "generation_latency_seconds": 1.2317780090961605,
1858
+ "batch_wall_latency_seconds": 1.246546880924143,
1859
+ "amortized_generation_ms_per_sample": 1231.7780090961605,
1860
+ "amortized_batch_wall_ms_per_sample": 1246.546880924143,
1861
+ "outputs": [
1862
+ {
1863
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[0,80,420,910],\"score\":0.95},{\"bbox_2d\":[460,0,760,200],\"score\":0.85}]}",
1864
+ "valid_prediction": {
1865
+ "objects": [
1866
+ {
1867
+ "bbox_2d": [
1868
+ 0,
1869
+ 80,
1870
+ 420,
1871
+ 910
1872
+ ],
1873
+ "score": 0.95
1874
+ },
1875
+ {
1876
+ "bbox_2d": [
1877
+ 460,
1878
+ 0,
1879
+ 760,
1880
+ 200
1881
+ ],
1882
+ "score": 0.85
1883
+ }
1884
+ ]
1885
+ },
1886
+ "error": null,
1887
+ "finish_reason": "stop",
1888
+ "completion_tokens": 56,
1889
+ "prompt_tokens": 526
1890
+ }
1891
+ ]
1892
+ },
1893
+ {
1894
+ "sample_ids": [
1895
+ "testA-49055-135086"
1896
+ ],
1897
+ "batch_size": 1,
1898
+ "generation_latency_seconds": 1.8898930319119245,
1899
+ "batch_wall_latency_seconds": 1.899978275061585,
1900
+ "amortized_generation_ms_per_sample": 1889.8930319119245,
1901
+ "amortized_batch_wall_ms_per_sample": 1899.978275061585,
1902
+ "outputs": [
1903
+ {
1904
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[120,140,320,980] ,\"score\":0.95},{\"bbox_2d\":[320,160,620,970] ,\"score\":0.94},{\"bbox_2d\":[620,250,860,980] ,\"score\":0.93}]}",
1905
+ "valid_prediction": {
1906
+ "objects": [
1907
+ {
1908
+ "bbox_2d": [
1909
+ 120,
1910
+ 140,
1911
+ 320,
1912
+ 980
1913
+ ],
1914
+ "score": 0.95
1915
+ },
1916
+ {
1917
+ "bbox_2d": [
1918
+ 320,
1919
+ 160,
1920
+ 620,
1921
+ 970
1922
+ ],
1923
+ "score": 0.94
1924
+ },
1925
+ {
1926
+ "bbox_2d": [
1927
+ 620,
1928
+ 250,
1929
+ 860,
1930
+ 980
1931
+ ],
1932
+ "score": 0.93
1933
+ }
1934
+ ]
1935
+ },
1936
+ "error": null,
1937
+ "finish_reason": "stop",
1938
+ "completion_tokens": 92,
1939
+ "prompt_tokens": 526
1940
+ }
1941
+ ]
1942
+ },
1943
+ {
1944
+ "sample_ids": [
1945
+ "testA-7857-21644"
1946
+ ],
1947
+ "batch_size": 1,
1948
+ "generation_latency_seconds": 9.8525026650168,
1949
+ "batch_wall_latency_seconds": 9.865253204014152,
1950
+ "amortized_generation_ms_per_sample": 9852.5026650168,
1951
+ "amortized_batch_wall_ms_per_sample": 9865.253204014152,
1952
+ "outputs": [
1953
+ {
1954
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[650 ",
1955
+ "valid_prediction": null,
1956
+ "error": "Expecting ',' delimiter: line 1 column 529 (char 528)",
1957
+ "finish_reason": "length",
1958
+ "completion_tokens": 512,
1959
+ "prompt_tokens": 527
1960
+ }
1961
+ ]
1962
+ },
1963
+ {
1964
+ "sample_ids": [
1965
+ "testB-79645-219273"
1966
+ ],
1967
+ "batch_size": 1,
1968
+ "generation_latency_seconds": 2.319699679967016,
1969
+ "batch_wall_latency_seconds": 2.332257404923439,
1970
+ "amortized_generation_ms_per_sample": 2319.699679967016,
1971
+ "amortized_batch_wall_ms_per_sample": 2332.257404923439,
1972
+ "outputs": [
1973
+ {
1974
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[120,420,540,700] ,\"score\":0.95},{\"bbox_2d\":[540,400,780,690] ,\"score\":0.94},{\"bbox_2d\":[0,450,120,670] ,\"score\":0.93},{\"bbox_2d\":[140,460,260,600] ,\"score\":0.92}]}",
1975
+ "valid_prediction": {
1976
+ "objects": [
1977
+ {
1978
+ "bbox_2d": [
1979
+ 120,
1980
+ 420,
1981
+ 540,
1982
+ 700
1983
+ ],
1984
+ "score": 0.95
1985
+ },
1986
+ {
1987
+ "bbox_2d": [
1988
+ 540,
1989
+ 400,
1990
+ 780,
1991
+ 690
1992
+ ],
1993
+ "score": 0.94
1994
+ },
1995
+ {
1996
+ "bbox_2d": [
1997
+ 0,
1998
+ 450,
1999
+ 120,
2000
+ 670
2001
+ ],
2002
+ "score": 0.93
2003
+ },
2004
+ {
2005
+ "bbox_2d": [
2006
+ 140,
2007
+ 460,
2008
+ 260,
2009
+ 600
2010
+ ],
2011
+ "score": 0.92
2012
+ }
2013
+ ]
2014
+ },
2015
+ "error": null,
2016
+ "finish_reason": "stop",
2017
+ "completion_tokens": 119,
2018
+ "prompt_tokens": 521
2019
+ }
2020
+ ]
2021
+ },
2022
+ {
2023
+ "sample_ids": [
2024
+ "testB-52921-145707"
2025
+ ],
2026
+ "batch_size": 1,
2027
+ "generation_latency_seconds": 10.00177818897646,
2028
+ "batch_wall_latency_seconds": 10.020861182012595,
2029
+ "amortized_generation_ms_per_sample": 10001.77818897646,
2030
+ "amortized_batch_wall_ms_per_sample": 10020.861182012595,
2031
+ "outputs": [
2032
+ {
2033
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[380 ",
2034
+ "valid_prediction": null,
2035
+ "error": "Expecting ',' delimiter: line 1 column 529 (char 528)",
2036
+ "finish_reason": "length",
2037
+ "completion_tokens": 512,
2038
+ "prompt_tokens": 531
2039
+ }
2040
+ ]
2041
+ },
2042
+ {
2043
+ "sample_ids": [
2044
+ "testB-69479-191562"
2045
+ ],
2046
+ "batch_size": 1,
2047
+ "generation_latency_seconds": 0.6840293509885669,
2048
+ "batch_wall_latency_seconds": 0.70091468305327,
2049
+ "amortized_generation_ms_per_sample": 684.0293509885669,
2050
+ "amortized_batch_wall_ms_per_sample": 700.91468305327,
2051
+ "outputs": [
2052
+ {
2053
+ "raw_response": "{\"objects\":[{\"bbox_2d\":[0,30,974,986],\"score\":0.95}]}",
2054
+ "valid_prediction": {
2055
+ "objects": [
2056
+ {
2057
+ "bbox_2d": [
2058
+ 0,
2059
+ 30,
2060
+ 974,
2061
+ 986
2062
+ ],
2063
+ "score": 0.95
2064
+ }
2065
+ ]
2066
+ },
2067
+ "error": null,
2068
+ "finish_reason": "stop",
2069
+ "completion_tokens": 30,
2070
+ "prompt_tokens": 522
2071
+ }
2072
+ ]
2073
+ },
2074
+ {
2075
+ "sample_ids": [
2076
+ "testA-69466-191524"
2077
+ ],
2078
+ "batch_size": 1,
2079
+ "generation_latency_seconds": 0.24351457098964602,
2080
+ "batch_wall_latency_seconds": 0.2538871930446476,
2081
+ "amortized_generation_ms_per_sample": 243.51457098964602,
2082
+ "amortized_batch_wall_ms_per_sample": 253.88719304464757,
2083
+ "outputs": [
2084
+ {
2085
+ "raw_response": "{\"objects\":[]}",
2086
+ "valid_prediction": {
2087
+ "objects": []
2088
+ },
2089
+ "error": null,
2090
+ "finish_reason": "stop",
2091
+ "completion_tokens": 6,
2092
+ "prompt_tokens": 522
2093
+ }
2094
+ ]
2095
+ }
2096
+ ],
2097
+ "mean_generation_ms_per_sample": 3652.817064514238,
2098
+ "median_generation_ms_per_sample": 1280.612550501246,
2099
+ "mean_batch_wall_ms_per_sample": 3667.173011115665,
2100
+ "median_batch_wall_ms_per_sample": 1289.789535512682,
2101
+ "mean_prompt_tokens": 517.609375,
2102
+ "mean_completion_tokens": 178.0625,
2103
+ "latency_scope": "A800 vLLM-0.18.0 native generation latency with the frozen task prompt and structured-output schema; model loading and warmup excluded. Batch wall time includes the real request/image preparation path.",
2104
+ "flops_scope": "This file measures latency only. Counted FLOPs are measured separately by the HF reference probe and are not claimed to be complete vLLM kernel FLOPs."
2105
+ }
releases/20260913_lr/paper_data/main_table_cost_20260912/components/qwen_flops.json ADDED
@@ -0,0 +1,341 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "complete",
3
+ "model_kind": "Qwen3.8",
4
+ "task": "construction",
5
+ "model_path": "/mnt/HithinkOmniSSD/user_workspace/caisihang/checkpoints/Qwen3.8-27B-FP8",
6
+ "sample_count": 8,
7
+ "records": [
8
+ {
9
+ "sample_id": "0000184",
10
+ "prompt_tokens_in_reference_tensor": 1705,
11
+ "input_shapes": {
12
+ "input_ids": [
13
+ 1,
14
+ 1705
15
+ ],
16
+ "attention_mask": [
17
+ 1,
18
+ 1705
19
+ ],
20
+ "mm_token_type_ids": [
21
+ 1,
22
+ 1705
23
+ ],
24
+ "pixel_values": [
25
+ 3888,
26
+ 1536
27
+ ],
28
+ "image_grid_thw": [
29
+ 1,
30
+ 3
31
+ ]
32
+ },
33
+ "prefill_flops_counted": 90083398713024,
34
+ "decode_one_token_flops_counted": 51922600128,
35
+ "prefill_gflops_counted": 90083.398713024,
36
+ "decode_one_token_gflops_counted": 51.922600128,
37
+ "prefill_operators": {
38
+ "aten.convolution": 20475543552,
39
+ "aten.addmm": 3283626885120,
40
+ "aten.bmm": 3741356981952,
41
+ "aten.mm": 83037939302400
42
+ },
43
+ "decode_operators": {
44
+ "aten.bmm": 670826688,
45
+ "aten.mm": 51243909120,
46
+ "aten.convolution": 7864320
47
+ }
48
+ },
49
+ {
50
+ "sample_id": "0001955",
51
+ "prompt_tokens_in_reference_tensor": 1705,
52
+ "input_shapes": {
53
+ "input_ids": [
54
+ 1,
55
+ 1705
56
+ ],
57
+ "attention_mask": [
58
+ 1,
59
+ 1705
60
+ ],
61
+ "mm_token_type_ids": [
62
+ 1,
63
+ 1705
64
+ ],
65
+ "pixel_values": [
66
+ 3888,
67
+ 1536
68
+ ],
69
+ "image_grid_thw": [
70
+ 1,
71
+ 3
72
+ ]
73
+ },
74
+ "prefill_flops_counted": 90083398713024,
75
+ "decode_one_token_flops_counted": 51922600128,
76
+ "prefill_gflops_counted": 90083.398713024,
77
+ "decode_one_token_gflops_counted": 51.922600128,
78
+ "prefill_operators": {
79
+ "aten.convolution": 20475543552,
80
+ "aten.addmm": 3283626885120,
81
+ "aten.bmm": 3741356981952,
82
+ "aten.mm": 83037939302400
83
+ },
84
+ "decode_operators": {
85
+ "aten.bmm": 670826688,
86
+ "aten.mm": 51243909120,
87
+ "aten.convolution": 7864320
88
+ }
89
+ },
90
+ {
91
+ "sample_id": "0000840",
92
+ "prompt_tokens_in_reference_tensor": 1705,
93
+ "input_shapes": {
94
+ "input_ids": [
95
+ 1,
96
+ 1705
97
+ ],
98
+ "attention_mask": [
99
+ 1,
100
+ 1705
101
+ ],
102
+ "mm_token_type_ids": [
103
+ 1,
104
+ 1705
105
+ ],
106
+ "pixel_values": [
107
+ 3888,
108
+ 1536
109
+ ],
110
+ "image_grid_thw": [
111
+ 1,
112
+ 3
113
+ ]
114
+ },
115
+ "prefill_flops_counted": 90083398713024,
116
+ "decode_one_token_flops_counted": 51922600128,
117
+ "prefill_gflops_counted": 90083.398713024,
118
+ "decode_one_token_gflops_counted": 51.922600128,
119
+ "prefill_operators": {
120
+ "aten.convolution": 20475543552,
121
+ "aten.addmm": 3283626885120,
122
+ "aten.bmm": 3741356981952,
123
+ "aten.mm": 83037939302400
124
+ },
125
+ "decode_operators": {
126
+ "aten.bmm": 670826688,
127
+ "aten.mm": 51243909120,
128
+ "aten.convolution": 7864320
129
+ }
130
+ },
131
+ {
132
+ "sample_id": "0003887",
133
+ "prompt_tokens_in_reference_tensor": 1705,
134
+ "input_shapes": {
135
+ "input_ids": [
136
+ 1,
137
+ 1705
138
+ ],
139
+ "attention_mask": [
140
+ 1,
141
+ 1705
142
+ ],
143
+ "mm_token_type_ids": [
144
+ 1,
145
+ 1705
146
+ ],
147
+ "pixel_values": [
148
+ 3888,
149
+ 1536
150
+ ],
151
+ "image_grid_thw": [
152
+ 1,
153
+ 3
154
+ ]
155
+ },
156
+ "prefill_flops_counted": 90083398713024,
157
+ "decode_one_token_flops_counted": 51922600128,
158
+ "prefill_gflops_counted": 90083.398713024,
159
+ "decode_one_token_gflops_counted": 51.922600128,
160
+ "prefill_operators": {
161
+ "aten.convolution": 20475543552,
162
+ "aten.addmm": 3283626885120,
163
+ "aten.bmm": 3741356981952,
164
+ "aten.mm": 83037939302400
165
+ },
166
+ "decode_operators": {
167
+ "aten.bmm": 670826688,
168
+ "aten.mm": 51243909120,
169
+ "aten.convolution": 7864320
170
+ }
171
+ },
172
+ {
173
+ "sample_id": "0003433",
174
+ "prompt_tokens_in_reference_tensor": 1705,
175
+ "input_shapes": {
176
+ "input_ids": [
177
+ 1,
178
+ 1705
179
+ ],
180
+ "attention_mask": [
181
+ 1,
182
+ 1705
183
+ ],
184
+ "mm_token_type_ids": [
185
+ 1,
186
+ 1705
187
+ ],
188
+ "pixel_values": [
189
+ 3888,
190
+ 1536
191
+ ],
192
+ "image_grid_thw": [
193
+ 1,
194
+ 3
195
+ ]
196
+ },
197
+ "prefill_flops_counted": 90083398713024,
198
+ "decode_one_token_flops_counted": 51922600128,
199
+ "prefill_gflops_counted": 90083.398713024,
200
+ "decode_one_token_gflops_counted": 51.922600128,
201
+ "prefill_operators": {
202
+ "aten.convolution": 20475543552,
203
+ "aten.addmm": 3283626885120,
204
+ "aten.bmm": 3741356981952,
205
+ "aten.mm": 83037939302400
206
+ },
207
+ "decode_operators": {
208
+ "aten.bmm": 670826688,
209
+ "aten.mm": 51243909120,
210
+ "aten.convolution": 7864320
211
+ }
212
+ },
213
+ {
214
+ "sample_id": "0001613",
215
+ "prompt_tokens_in_reference_tensor": 1705,
216
+ "input_shapes": {
217
+ "input_ids": [
218
+ 1,
219
+ 1705
220
+ ],
221
+ "attention_mask": [
222
+ 1,
223
+ 1705
224
+ ],
225
+ "mm_token_type_ids": [
226
+ 1,
227
+ 1705
228
+ ],
229
+ "pixel_values": [
230
+ 3888,
231
+ 1536
232
+ ],
233
+ "image_grid_thw": [
234
+ 1,
235
+ 3
236
+ ]
237
+ },
238
+ "prefill_flops_counted": 90083398713024,
239
+ "decode_one_token_flops_counted": 51922600128,
240
+ "prefill_gflops_counted": 90083.398713024,
241
+ "decode_one_token_gflops_counted": 51.922600128,
242
+ "prefill_operators": {
243
+ "aten.convolution": 20475543552,
244
+ "aten.addmm": 3283626885120,
245
+ "aten.bmm": 3741356981952,
246
+ "aten.mm": 83037939302400
247
+ },
248
+ "decode_operators": {
249
+ "aten.bmm": 670826688,
250
+ "aten.mm": 51243909120,
251
+ "aten.convolution": 7864320
252
+ }
253
+ },
254
+ {
255
+ "sample_id": "0002591",
256
+ "prompt_tokens_in_reference_tensor": 1705,
257
+ "input_shapes": {
258
+ "input_ids": [
259
+ 1,
260
+ 1705
261
+ ],
262
+ "attention_mask": [
263
+ 1,
264
+ 1705
265
+ ],
266
+ "mm_token_type_ids": [
267
+ 1,
268
+ 1705
269
+ ],
270
+ "pixel_values": [
271
+ 3888,
272
+ 1536
273
+ ],
274
+ "image_grid_thw": [
275
+ 1,
276
+ 3
277
+ ]
278
+ },
279
+ "prefill_flops_counted": 90083398713024,
280
+ "decode_one_token_flops_counted": 51922600128,
281
+ "prefill_gflops_counted": 90083.398713024,
282
+ "decode_one_token_gflops_counted": 51.922600128,
283
+ "prefill_operators": {
284
+ "aten.convolution": 20475543552,
285
+ "aten.addmm": 3283626885120,
286
+ "aten.bmm": 3741356981952,
287
+ "aten.mm": 83037939302400
288
+ },
289
+ "decode_operators": {
290
+ "aten.bmm": 670826688,
291
+ "aten.mm": 51243909120,
292
+ "aten.convolution": 7864320
293
+ }
294
+ },
295
+ {
296
+ "sample_id": "0003105",
297
+ "prompt_tokens_in_reference_tensor": 1705,
298
+ "input_shapes": {
299
+ "input_ids": [
300
+ 1,
301
+ 1705
302
+ ],
303
+ "attention_mask": [
304
+ 1,
305
+ 1705
306
+ ],
307
+ "mm_token_type_ids": [
308
+ 1,
309
+ 1705
310
+ ],
311
+ "pixel_values": [
312
+ 3888,
313
+ 1536
314
+ ],
315
+ "image_grid_thw": [
316
+ 1,
317
+ 3
318
+ ]
319
+ },
320
+ "prefill_flops_counted": 90083398713024,
321
+ "decode_one_token_flops_counted": 51922600128,
322
+ "prefill_gflops_counted": 90083.398713024,
323
+ "decode_one_token_gflops_counted": 51.922600128,
324
+ "prefill_operators": {
325
+ "aten.convolution": 20475543552,
326
+ "aten.addmm": 3283626885120,
327
+ "aten.bmm": 3741356981952,
328
+ "aten.mm": 83037939302400
329
+ },
330
+ "decode_operators": {
331
+ "aten.bmm": 670826688,
332
+ "aten.mm": 51243909120,
333
+ "aten.convolution": 7864320
334
+ }
335
+ }
336
+ ],
337
+ "mean_prefill_gflops_counted": 90083.398713024,
338
+ "mean_decode_one_token_gflops_counted": 51.922600128,
339
+ "scope": "HF reference execution on the A800 with real task inputs; dense products/convolutions/supported attention counted with multiply-add=2 FLOPs. vLLM custom kernels, elementwise/normalization/nonlinearities and other unsupported operations are not complete.",
340
+ "answer_generation": "not performed; one known-token decode probe only"
341
+ }
releases/20260913_lr/paper_data/main_table_cost_20260912/components/qwen_latency.json ADDED
The diff for this file is too large to render. See raw diff
 
releases/20260913_lr/paper_data/main_table_cost_20260912/components/yolo26x_router.json ADDED
@@ -0,0 +1,238 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "complete",
3
+ "run": "shared_12_yolo26x_Qwen3.8",
4
+ "config": {
5
+ "fraction": 1.0,
6
+ "seed": 42,
7
+ "freeze_encoder": true,
8
+ "fusion": "concat",
9
+ "harm_cost": 1.0,
10
+ "batch_size": 256,
11
+ "weight_decay": 0.01,
12
+ "max_epochs": 50,
13
+ "patience": 8,
14
+ "cpu_threads": 1,
15
+ "student": "8M",
16
+ "representation_dim": 128,
17
+ "lr": 0.003,
18
+ "target": "four_state",
19
+ "strategy": "natural",
20
+ "hidden_percent": 90,
21
+ "expert": "yolo26x",
22
+ "task": "construction",
23
+ "endpoint": "Qwen3.8",
24
+ "branches": [
25
+ "image",
26
+ "hidden",
27
+ "confidence",
28
+ "output"
29
+ ],
30
+ "hidden_layer": "feature_blocks__layer_16",
31
+ "id": "shared_12_yolo26x_Qwen3.8",
32
+ "stage": "shared_mandatory_inputs"
33
+ },
34
+ "checkpoint": "/mnt/HithinkOmniSSD/user_workspace/caisihang/research/ICASSP_ModelCollaboration/outputs/router_shared_image_hidden_20260910/runs/shared_12_yolo26x_Qwen3.8/best.pt",
35
+ "score": "logit_rescue-logit_harm",
36
+ "test_rows_recomputed": 3004,
37
+ "selected_decisions_match_main": true,
38
+ "max_score_abs_difference": 0.0,
39
+ "actual_call_fraction": 0.4876830892143808,
40
+ "sample_count": 64,
41
+ "sample_ids": [
42
+ "0002628",
43
+ "0003242",
44
+ "0001568",
45
+ "0001119",
46
+ "0003648",
47
+ "0000187",
48
+ "0001891",
49
+ "0001689",
50
+ "0003372",
51
+ "0002549",
52
+ "0000915",
53
+ "0003633",
54
+ "0004424",
55
+ "0001627",
56
+ "0001562",
57
+ "0002018",
58
+ "0000254",
59
+ "0002719",
60
+ "0002730",
61
+ "0004413",
62
+ "0000891",
63
+ "0000150",
64
+ "0003223",
65
+ "0004731",
66
+ "0001587",
67
+ "0000483",
68
+ "0001116",
69
+ "0003376",
70
+ "0001225",
71
+ "0003082",
72
+ "0004447",
73
+ "0000283",
74
+ "0000563",
75
+ "0001254",
76
+ "0004759",
77
+ "0001868",
78
+ "0001860",
79
+ "0004087",
80
+ "0002962",
81
+ "0001802",
82
+ "0002973",
83
+ "0002648",
84
+ "0004350",
85
+ "0004377",
86
+ "0004809",
87
+ "0003694",
88
+ "0001572",
89
+ "0001550",
90
+ "0000213",
91
+ "0002266",
92
+ "0002445",
93
+ "0004513",
94
+ "0002124",
95
+ "0004391",
96
+ "0003679",
97
+ "0002902",
98
+ "0002508",
99
+ "0004767",
100
+ "0003840",
101
+ "0003498",
102
+ "0001579",
103
+ "0001748",
104
+ "0004706",
105
+ "0004387"
106
+ ],
107
+ "batch_size": 1,
108
+ "latency_ms": [
109
+ 17.35539303626865,
110
+ 13.579263002611697,
111
+ 15.319477999582887,
112
+ 14.879688038490713,
113
+ 24.25052202306688,
114
+ 20.667891018092632,
115
+ 20.015343907289207,
116
+ 19.221558002755046,
117
+ 20.937222987413406,
118
+ 26.26270893961191,
119
+ 17.899635946378112,
120
+ 16.57471200451255,
121
+ 18.844021018594503,
122
+ 21.3212959934026,
123
+ 19.545868039131165,
124
+ 64.57463896367699,
125
+ 20.962884067557752,
126
+ 24.004184058867395,
127
+ 20.19065897911787,
128
+ 18.50342599209398,
129
+ 19.576229038648307,
130
+ 25.253725005313754,
131
+ 21.430211956612766,
132
+ 25.169604923576117,
133
+ 17.8982320940122,
134
+ 21.415626979433,
135
+ 22.9903319850564,
136
+ 229.59673800505698,
137
+ 19.89229000173509,
138
+ 18.98771501146257,
139
+ 18.157949903979897,
140
+ 22.423187969252467,
141
+ 19.864327972754836,
142
+ 21.502194926142693,
143
+ 18.544791964814067,
144
+ 50.5554600385949,
145
+ 20.046600024215877,
146
+ 16.299219918437302,
147
+ 18.668118049390614,
148
+ 24.017272982746363,
149
+ 24.73958197515458,
150
+ 18.68593308608979,
151
+ 20.250934991054237,
152
+ 23.676551994867623,
153
+ 23.003229987807572,
154
+ 24.6098650386557,
155
+ 21.308812079951167,
156
+ 17.95686304103583,
157
+ 18.23423698078841,
158
+ 14.343310962431133,
159
+ 25.8766250917688,
160
+ 18.20633898023516,
161
+ 30.77617404051125,
162
+ 19.3530690157786,
163
+ 15.836970997042954,
164
+ 19.045730936340988,
165
+ 16.162591986358166,
166
+ 25.458221905864775,
167
+ 18.277363968081772,
168
+ 52.543785073794425,
169
+ 22.9536690749228,
170
+ 17.81571004539728,
171
+ 18.550690030679107,
172
+ 21.854626946151257
173
+ ],
174
+ "mean_latency_ms_per_input": 25.261267359383055,
175
+ "head_mean_latency_ms": 0.486134999897331,
176
+ "gflops_per_input": [
177
+ 3.183754752,
178
+ 3.183754752,
179
+ 3.183754752,
180
+ 3.183754752,
181
+ 3.183754752,
182
+ 3.183754752,
183
+ 3.183754752,
184
+ 3.183754752
185
+ ],
186
+ "mean_gflops_per_input": 3.183754752,
187
+ "operators": [
188
+ {
189
+ "aten.convolution": 228802560,
190
+ "aten.bmm": 679257600,
191
+ "aten._scaled_dot_product_flash_attention": 236027904,
192
+ "aten.addmm": 2039666688
193
+ },
194
+ {
195
+ "aten.convolution": 228802560,
196
+ "aten.bmm": 679257600,
197
+ "aten._scaled_dot_product_flash_attention": 236027904,
198
+ "aten.addmm": 2039666688
199
+ },
200
+ {
201
+ "aten.convolution": 228802560,
202
+ "aten.bmm": 679257600,
203
+ "aten._scaled_dot_product_flash_attention": 236027904,
204
+ "aten.addmm": 2039666688
205
+ },
206
+ {
207
+ "aten.convolution": 228802560,
208
+ "aten.bmm": 679257600,
209
+ "aten._scaled_dot_product_flash_attention": 236027904,
210
+ "aten.addmm": 2039666688
211
+ },
212
+ {
213
+ "aten.convolution": 228802560,
214
+ "aten.bmm": 679257600,
215
+ "aten._scaled_dot_product_flash_attention": 236027904,
216
+ "aten.addmm": 2039666688
217
+ },
218
+ {
219
+ "aten.convolution": 228802560,
220
+ "aten.bmm": 679257600,
221
+ "aten._scaled_dot_product_flash_attention": 236027904,
222
+ "aten.addmm": 2039666688
223
+ },
224
+ {
225
+ "aten.convolution": 228802560,
226
+ "aten.bmm": 679257600,
227
+ "aten._scaled_dot_product_flash_attention": 236027904,
228
+ "aten.addmm": 2039666688
229
+ },
230
+ {
231
+ "aten.convolution": 228802560,
232
+ "aten.bmm": 679257600,
233
+ "aten._scaled_dot_product_flash_attention": 236027904,
234
+ "aten.addmm": 2039666688
235
+ }
236
+ ],
237
+ "scope": "image loading, bilinear resize, tokenize when used, two image slots (including gray placeholder), active distilled encoders, feature assembly and original routing head; cached specialist features; supported FLOPs only"
238
+ }
releases/20260913_lr/paper_data/main_table_cost_20260912/components/yolo26x_specialist.json ADDED
@@ -0,0 +1,254 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "component": "specialist",
3
+ "expert": "yolo26x",
4
+ "task": "construction",
5
+ "device": "cuda:0",
6
+ "checkpoint": "/mnt/HithinkOmniSSD/user_workspace/caisihang/research/ICASSP_ModelCollaboration/models/construction/yolo26x.pt",
7
+ "sample_count": 64,
8
+ "batch_size": 1,
9
+ "sample_ids": [
10
+ "0000944",
11
+ "0003511",
12
+ "0003365",
13
+ "0001351",
14
+ "0002149",
15
+ "0003368",
16
+ "0000176",
17
+ "0004487",
18
+ "0000013",
19
+ "0003583",
20
+ "0000758",
21
+ "0001291",
22
+ "0001451",
23
+ "0002755",
24
+ "0003088",
25
+ "0004538",
26
+ "0004324",
27
+ "0000516",
28
+ "0003893",
29
+ "0003475",
30
+ "0002137",
31
+ "0003272",
32
+ "0001711",
33
+ "0003262",
34
+ "0001470",
35
+ "0002145",
36
+ "0002662",
37
+ "0000756",
38
+ "0001631",
39
+ "0001924",
40
+ "0002496",
41
+ "0001443",
42
+ "0004053",
43
+ "0002168",
44
+ "0001881",
45
+ "0001666",
46
+ "0000304",
47
+ "0001252",
48
+ "0004844",
49
+ "0000949",
50
+ "0003430",
51
+ "0003807",
52
+ "0004199",
53
+ "0002538",
54
+ "0002091",
55
+ "0001569",
56
+ "0003048",
57
+ "0002854",
58
+ "0000380",
59
+ "0001234",
60
+ "0001484",
61
+ "0003180",
62
+ "0000824",
63
+ "0003784",
64
+ "0000894",
65
+ "0004168",
66
+ "0000564",
67
+ "0004296",
68
+ "0003054",
69
+ "0001583",
70
+ "0003386",
71
+ "0000205",
72
+ "0004214",
73
+ "0003625"
74
+ ],
75
+ "model_load_seconds_excluded_from_latency": true,
76
+ "forward_latency_ms_per_sample": [
77
+ 41.741790771484375,
78
+ 51.50377655029297,
79
+ 78.82457733154297,
80
+ 87.29910278320312,
81
+ 49.47385787963867,
82
+ 49.40816116333008,
83
+ 168.38259887695312,
84
+ 42.57542419433594,
85
+ 40.342529296875,
86
+ 54.95475387573242,
87
+ 48.86323165893555,
88
+ 40.684993743896484,
89
+ 282.5484619140625,
90
+ 47.938270568847656,
91
+ 45.019649505615234,
92
+ 268.04327392578125,
93
+ 51.39228820800781,
94
+ 41.36457443237305,
95
+ 54.01919937133789,
96
+ 43.2599983215332,
97
+ 47.13923263549805,
98
+ 41.1484489440918,
99
+ 35.99542236328125,
100
+ 80.46995544433594,
101
+ 83.43251037597656,
102
+ 48.88550567626953,
103
+ 171.4989776611328,
104
+ 58.917728424072266,
105
+ 46.90486526489258,
106
+ 56.416385650634766,
107
+ 44.49577713012695,
108
+ 49.42009735107422,
109
+ 43.6354866027832,
110
+ 55.22761535644531,
111
+ 48.02000045776367,
112
+ 39.44412612915039,
113
+ 30.26905632019043,
114
+ 492.74456787109375,
115
+ 45.74156951904297,
116
+ 39.223392486572266,
117
+ 50.18822479248047,
118
+ 43.890113830566406,
119
+ 51.966495513916016,
120
+ 47.814815521240234,
121
+ 54.13961410522461,
122
+ 39.215328216552734,
123
+ 56.004703521728516,
124
+ 39.528831481933594,
125
+ 396.5307312011719,
126
+ 83.4832992553711,
127
+ 50.0503044128418,
128
+ 153.1714630126953,
129
+ 61.07219314575195,
130
+ 52.57244873046875,
131
+ 42.75983810424805,
132
+ 38.4617919921875,
133
+ 44.08700942993164,
134
+ 53.36006546020508,
135
+ 45.519649505615234,
136
+ 290.8823547363281,
137
+ 48.10774230957031,
138
+ 53.34476852416992,
139
+ 51.016319274902344,
140
+ 42.9719352722168
141
+ ],
142
+ "end_to_end_latency_ms_per_sample": [
143
+ 35.17372906208038,
144
+ 48.07086195796728,
145
+ 46.07532289810479,
146
+ 45.34715902991593,
147
+ 46.541396994143724,
148
+ 45.594630064442754,
149
+ 45.38595397025347,
150
+ 39.57494895439595,
151
+ 39.812199072912335,
152
+ 51.36631103232503,
153
+ 42.81833197455853,
154
+ 33.49219704978168,
155
+ 37.3103239107877,
156
+ 38.96563302259892,
157
+ 40.327758993953466,
158
+ 44.32409000582993,
159
+ 43.25311293359846,
160
+ 33.99710892699659,
161
+ 49.12354901898652,
162
+ 45.043769059702754,
163
+ 47.554407035931945,
164
+ 37.903546006418765,
165
+ 32.56980108562857,
166
+ 35.192839917726815,
167
+ 39.34567596297711,
168
+ 46.67255701497197,
169
+ 49.70169102307409,
170
+ 52.48237494379282,
171
+ 43.415413005277514,
172
+ 48.36420400533825,
173
+ 48.96194895263761,
174
+ 51.6092429170385,
175
+ 37.311508087441325,
176
+ 50.858690054155886,
177
+ 43.94579504150897,
178
+ 36.09434806276113,
179
+ 26.66287892498076,
180
+ 41.9749760767445,
181
+ 34.900374012067914,
182
+ 39.04577391222119,
183
+ 51.60050210542977,
184
+ 46.00996105000377,
185
+ 53.40714694466442,
186
+ 51.78673507180065,
187
+ 54.96632098220289,
188
+ 46.47863702848554,
189
+ 47.48446703888476,
190
+ 36.593522992916405,
191
+ 37.32479899190366,
192
+ 26.43477905075997,
193
+ 39.97986298054457,
194
+ 55.87899999227375,
195
+ 40.821335976943374,
196
+ 45.88699492160231,
197
+ 40.707007981836796,
198
+ 34.627774031832814,
199
+ 43.4188200160861,
200
+ 49.56604307517409,
201
+ 39.82124803587794,
202
+ 249.81214595027268,
203
+ 38.2277499884367,
204
+ 48.29735495150089,
205
+ 47.6501100929454,
206
+ 40.76927900314331
207
+ ],
208
+ "mean_forward_latency_ms_per_sample": 78.70017620921135,
209
+ "median_forward_latency_ms_per_sample": 49.41412925720215,
210
+ "mean_end_to_end_latency_ms_per_sample": 46.30809425361804,
211
+ "median_end_to_end_latency_ms_per_sample": 43.68230752879754,
212
+ "flop_profiles": [
213
+ {
214
+ "batch_size": 1,
215
+ "flops_counted": 116518072320,
216
+ "gflops_counted_per_sample": 116.51807232,
217
+ "operators": {
218
+ "aten.convolution": 116319006720,
219
+ "aten.bmm": 199065600
220
+ }
221
+ },
222
+ {
223
+ "batch_size": 1,
224
+ "flops_counted": 145709798400,
225
+ "gflops_counted_per_sample": 145.7097984,
226
+ "operators": {
227
+ "aten.convolution": 145398758400,
228
+ "aten.bmm": 311040000
229
+ }
230
+ },
231
+ {
232
+ "batch_size": 1,
233
+ "flops_counted": 145709798400,
234
+ "gflops_counted_per_sample": 145.7097984,
235
+ "operators": {
236
+ "aten.convolution": 145398758400,
237
+ "aten.bmm": 311040000
238
+ }
239
+ },
240
+ {
241
+ "batch_size": 1,
242
+ "flops_counted": 135976458240,
243
+ "gflops_counted_per_sample": 135.97645824,
244
+ "operators": {
245
+ "aten.convolution": 135705507840,
246
+ "aten.bmm": 270950400
247
+ }
248
+ }
249
+ ],
250
+ "mean_counted_gflops_per_sample": 135.97853184000002,
251
+ "flops_scope": "GPU FlopCounterMode around the real Ultralytics predict path; multiply-add counted as 2 FLOPs. Elementwise, postprocessing and unsupported custom ops may be missing.",
252
+ "latency_scope": "A800 wall-clock latency for real Ultralytics prediction including image loading/preprocess, forward and result postprocessing, excluding model load; batch processing uses the recorded batch size.",
253
+ "completed": true
254
+ }
releases/20260913_lr/paper_data/main_table_cost_20260912/cost_compact.pdf ADDED
Binary file (43.6 kB). View file
 
releases/20260913_lr/paper_data/main_table_cost_20260912/cost_compact.png ADDED

Git LFS Details

  • SHA256: c3e5c4bfdf5310e20a021a38d0faf8c2394864d2f65b05b76736154cf041dfc4
  • Pointer size: 131 Bytes
  • Size of remote file: 121 kB
releases/20260913_lr/paper_data/main_table_cost_20260912/cost_compact_verified.png ADDED
releases/20260913_lr/paper_data/main_table_cost_20260912/cost_summary.json ADDED
@@ -0,0 +1,75 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "components": {
3
+ "grefcoco|instancevg|MiniCPM": {
4
+ "specialist_gflops": 7.502680064,
5
+ "router_gflops": 3.642171392,
6
+ "large_gflops": 12916.147709832,
7
+ "specialist_ms": 46.08042127620138,
8
+ "router_ms": 13.180577338061994,
9
+ "large_ms": 3667.173011115665,
10
+ "large_samples": 64,
11
+ "flops_probes": 8,
12
+ "specialist_samples": 64,
13
+ "router_samples": 64,
14
+ "batch_size": 1
15
+ },
16
+ "construction|yolo26x|Qwen3.8": {
17
+ "specialist_gflops": 135.97853184000002,
18
+ "router_gflops": 3.183754752,
19
+ "large_gflops": 96874.712551641,
20
+ "specialist_ms": 46.30809425361804,
21
+ "router_ms": 25.261267359383055,
22
+ "large_ms": 9822.475872695577,
23
+ "large_samples": 64,
24
+ "flops_probes": 8,
25
+ "specialist_samples": 64,
26
+ "router_samples": 64,
27
+ "batch_size": 1
28
+ }
29
+ },
30
+ "selected_points": [
31
+ {
32
+ "pair": "grefcoco|instancevg|MiniCPM",
33
+ "method": "learned",
34
+ "nominal_validation_budget": 0.23,
35
+ "actual_test_call_fraction": 0.30904914499617164,
36
+ "test_metric": 0.6671865694921022,
37
+ "estimated_gflops_per_input": 4002.86925782384,
38
+ "estimated_time_ms_per_input": 1192.5976822525959
39
+ },
40
+ {
41
+ "pair": "grefcoco|instancevg|MiniCPM",
42
+ "method": "confidence",
43
+ "nominal_validation_budget": 0.65,
44
+ "actual_test_call_fraction": 0.5209426310864078,
45
+ "test_metric": 0.6187788900547316,
46
+ "estimated_gflops_per_input": 6736.074651524562,
47
+ "estimated_time_ms_per_input": 1956.4671783358606
48
+ },
49
+ {
50
+ "pair": "construction|yolo26x|Qwen3.8",
51
+ "method": "learned",
52
+ "nominal_validation_budget": 0.5,
53
+ "actual_test_call_fraction": 0.4876830892143808,
54
+ "test_metric": 0.29875800068449215,
55
+ "estimated_gflops_per_input": 47383.321370531434,
56
+ "estimated_time_ms_per_input": 4861.824738942901
57
+ },
58
+ {
59
+ "pair": "construction|yolo26x|Qwen3.8",
60
+ "method": "confidence",
61
+ "nominal_validation_budget": 0.93,
62
+ "actual_test_call_fraction": 0.9221038615179761,
63
+ "test_metric": 0.2597490004762105,
64
+ "estimated_gflops_per_input": 89464.5250591521,
65
+ "estimated_time_ms_per_input": 9103.651026133362
66
+ }
67
+ ],
68
+ "protocol": {
69
+ "method": "original main-table logit contrast; original checkpoints and validation thresholds; seed 42",
70
+ "large_timing": "64 measured examples; batch size one; same full task prompt; two separate warmup examples",
71
+ "cost": "serial component estimates, not a timed integrated service; includes active distilled encoders and routing head only for learned",
72
+ "flops": "multiply-add=2; supported operators; mean prefill (including visual embedding) plus average generated length times one-token decode probe",
73
+ "scope_limits": "mean call cost extrapolated across policies; no policy-specific input/output-length costs; specialist-feature extraction overhead is not separately measured; confidence threshold overhead not separately timed"
74
+ }
75
+ }