Publish latest paper, paired outcomes and verified reproduction materials
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +6 -0
- releases/20260913_lr/LATEST_RELEASE.md +50 -0
- releases/20260913_lr/MODELS_AND_DATA.md +51 -0
- releases/20260913_lr/REPRODUCE.md +50 -0
- releases/20260913_lr/appendix.pdf +3 -0
- releases/20260913_lr/main.pdf +3 -0
- releases/20260913_lr/paired_outcomes.tar.gz +3 -0
- releases/20260913_lr/paper_data/budget_diagnostics.json +1 -0
- releases/20260913_lr/paper_data/comparisons.json +0 -0
- releases/20260913_lr/paper_data/cost_20260911/construction_router.json +27 -0
- releases/20260913_lr/paper_data/cost_20260911/cost_summary.json +1036 -0
- releases/20260913_lr/paper_data/cost_20260911/fixed_budget_cost_performance.csv +55 -0
- releases/20260913_lr/paper_data/cost_20260911/full_cost_performance_curves.csv +0 -0
- releases/20260913_lr/paper_data/cost_20260911/grefcoco_router.json +27 -0
- releases/20260913_lr/paper_data/cost_20260911/instancevg_specialist.json +134 -0
- releases/20260913_lr/paper_data/cost_20260911/make_cost_analysis_20260911.py +457 -0
- releases/20260913_lr/paper_data/cost_20260911/minicpm_grefcoco.json +385 -0
- releases/20260913_lr/paper_data/cost_20260911/minicpm_grefcoco_flops.json +117 -0
- releases/20260913_lr/paper_data/cost_20260911/qwen_construction.json +385 -0
- releases/20260913_lr/paper_data/cost_20260911/qwen_construction_flops.json +177 -0
- releases/20260913_lr/paper_data/cost_20260911/router_architecture.json +37 -0
- releases/20260913_lr/paper_data/cost_20260911/yolo26x_specialist.json +134 -0
- releases/20260913_lr/paper_data/cost_20260911/成本性能补充报告.md +69 -0
- releases/20260913_lr/paper_data/cost_components.json +41 -0
- releases/20260913_lr/paper_data/cost_five_seeds.json +3299 -0
- releases/20260913_lr/paper_data/earlier_results.json +0 -0
- releases/20260913_lr/paper_data/fig2_audit/remote_counts.json +74 -0
- releases/20260913_lr/paper_data/fig2_audit/remote_counts.py +25 -0
- releases/20260913_lr/paper_data/five_seed_counts.json +7 -0
- releases/20260913_lr/paper_data/full_budget_results.json +0 -0
- releases/20260913_lr/paper_data/historical_results_v44.json +0 -0
- releases/20260913_lr/paper_data/joint_MiniCPM_comparison_20260910.json +58 -0
- releases/20260913_lr/paper_data/joint_Qwen3.8_comparison_20260910.json +58 -0
- releases/20260913_lr/paper_data/layout_verification.json +29 -0
- releases/20260913_lr/paper_data/lr_control_curves.json +0 -0
- releases/20260913_lr/paper_data/lr_control_summary.json +156 -0
- releases/20260913_lr/paper_data/lr_main_summary.json +210 -0
- releases/20260913_lr/paper_data/main_table_cost_20260912/build_figures.py +101 -0
- releases/20260913_lr/paper_data/main_table_cost_20260912/components/instancevg_router.json +239 -0
- releases/20260913_lr/paper_data/main_table_cost_20260912/components/instancevg_specialist.json +254 -0
- releases/20260913_lr/paper_data/main_table_cost_20260912/components/minicpm_flops.json +261 -0
- releases/20260913_lr/paper_data/main_table_cost_20260912/components/minicpm_latency.json +2105 -0
- releases/20260913_lr/paper_data/main_table_cost_20260912/components/qwen_flops.json +341 -0
- releases/20260913_lr/paper_data/main_table_cost_20260912/components/qwen_latency.json +0 -0
- releases/20260913_lr/paper_data/main_table_cost_20260912/components/yolo26x_router.json +238 -0
- releases/20260913_lr/paper_data/main_table_cost_20260912/components/yolo26x_specialist.json +254 -0
- releases/20260913_lr/paper_data/main_table_cost_20260912/cost_compact.pdf +0 -0
- releases/20260913_lr/paper_data/main_table_cost_20260912/cost_compact.png +3 -0
- releases/20260913_lr/paper_data/main_table_cost_20260912/cost_compact_verified.png +0 -0
- releases/20260913_lr/paper_data/main_table_cost_20260912/cost_summary.json +75 -0
.gitattributes
CHANGED
|
@@ -49,3 +49,9 @@ paper_results/figures/Fig1_editable.pptx filter=lfs diff=lfs merge=lfs -text
|
|
| 49 |
paper_results/figures/flops_performance.png filter=lfs diff=lfs merge=lfs -text
|
| 50 |
paper_results/figures/latency_performance.png filter=lfs diff=lfs merge=lfs -text
|
| 51 |
paper_results/figures/method.pdf filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 49 |
paper_results/figures/flops_performance.png filter=lfs diff=lfs merge=lfs -text
|
| 50 |
paper_results/figures/latency_performance.png filter=lfs diff=lfs merge=lfs -text
|
| 51 |
paper_results/figures/method.pdf filter=lfs diff=lfs merge=lfs -text
|
| 52 |
+
releases/20260913_lr/paper_data/main_table_cost_20260912/cost_compact.png filter=lfs diff=lfs merge=lfs -text
|
| 53 |
+
releases/20260913_lr/paper_data/main_table_cost_qwen_20260912/cost_compact.png filter=lfs diff=lfs merge=lfs -text
|
| 54 |
+
releases/20260913_lr/paper_data/reproduction/construction_image_path.jpg filter=lfs diff=lfs merge=lfs -text
|
| 55 |
+
releases/20260913_lr/paper_data/reproduction/nlvr2_image2.jpg filter=lfs diff=lfs merge=lfs -text
|
| 56 |
+
releases/20260913_lr/paper_data/test_results.json filter=lfs diff=lfs merge=lfs -text
|
| 57 |
+
releases/20260913_lr/paper_data/validation_manifest.json filter=lfs diff=lfs merge=lfs -text
|
releases/20260913_lr/LATEST_RELEASE.md
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Latest release: 20260913_lr
|
| 2 |
+
|
| 3 |
+
The release contains the latest paper results and the full experiment record. Download artifacts from `nohi191212/ModelCollaboration` on Hugging Face under `releases/20260913_lr/`.
|
| 4 |
+
|
| 5 |
+
## Weights and records
|
| 6 |
+
|
| 7 |
+
`full_budget_20260913_weights.tar.gz` contains exactly **80 router checkpoints**: 16 model pairs × five seeds for the main table. This release publishes no other router weights.
|
| 8 |
+
|
| 9 |
+
The study-specific `_records.tar.gz` archives retain configurations, training histories, validation/test outputs, curves and selection records for the experiments. Extract them into the same artifact directory; paths under `outputs/` are preserved. Earlier fixed-answer CSS/OvA adaptations are separate from the corrected K-class implementations.
|
| 10 |
+
|
| 11 |
+
Previously released backbone and specialist weights remain in the root-level `weights.tar`. External models and datasets are linked in `MODELS_AND_DATA.md`.
|
| 12 |
+
|
| 13 |
+
## Data
|
| 14 |
+
|
| 15 |
+
Eight `training_inputs_<expert>.tar.gz` files contain the train/validation/test arrays and example records for the latest 8M frozen-backbone setup. They contain image/text features, the selected specialist hidden state, confidence, output features, validity masks and labels. `input_manifest.json` records dimensions, sample counts and hidden-layer names. Set `ICASSP_DATA_ROOT` to the directory containing `training_inputs/` and this manifest.
|
| 16 |
+
|
| 17 |
+
`paired_outcomes.tar.gz` provides the paired correctness and task-metric sufficient statistics used for result replay. `paper_replay_runs.tar.gz` contains the 400 training runs used by the latest Tables 1 and 3, with raw logits and configurations. These two files are sufficient for `scripts/replay_latest_results.py`.
|
| 18 |
+
|
| 19 |
+
`additional_analysis_records.tar.gz` contains scoring explorations and cost records. `experiment_sources.tar.gz` preserves original experiment source snapshots, also included in the GitHub repository.
|
| 20 |
+
|
| 21 |
+
The latest `paper/data/` directory stores table and figure summaries. Main weights were selected over the full 0–100% validation curve. Table 3 retains its earlier training selection setting, as specified in the supplement and run configurations.
|
| 22 |
+
|
| 23 |
+
## Training examples
|
| 24 |
+
|
| 25 |
+
Install the dependencies in `requirements.txt` and a PyTorch build for your GPU. Commands below are run from the repository root. They start training only when invoked.
|
| 26 |
+
|
| 27 |
+
For normalized LR, download/extract a specialist's input archive and `input_manifest.json`, then run:
|
| 28 |
+
|
| 29 |
+
```bash
|
| 30 |
+
export ICASSP_DATA_ROOT="$PWD/artifacts/releases/20260913_lr"
|
| 31 |
+
python code/experiments/full_budget_train.py --source configs/full_budget/full_cub_MiniCPM_s2026 --output outputs/my_lr_run
|
| 32 |
+
```
|
| 33 |
+
|
| 34 |
+
For post-hoc deferral, extract its records archive, choose a saved `config.json`, and run:
|
| 35 |
+
|
| 36 |
+
```bash
|
| 37 |
+
python code/experiments/train_ltd.py --root "$PWD" --config "$ICASSP_DATA_ROOT/outputs/ltd_fixed_backbone_20260913/runs/cub_MiniCPM_posthoc_c0_s2026/config.json" --output outputs/my_posthoc_run --device cuda
|
| 38 |
+
```
|
| 39 |
+
|
| 40 |
+
Use `train_three.py` for a configuration from `three_state_20260913`, and `train_native.py` for a configuration from `ltd_kclass_native_20260913`. Configurations are supplied in the records and weight archives. The original objective experiment sources are in `experiments/source_snapshots/`; the five-seed objective trainer is also in `code/objectives/`.
|
| 41 |
+
|
| 42 |
+
To evaluate a newly trained post-hoc/three-state/native run, use its corresponding `evaluate_ltd.py`, `evaluate_three.py` or `evaluate_native.py` with `--help` for arguments. Input paths are resolved through `ICASSP_DATA_ROOT`.
|
| 43 |
+
|
| 44 |
+
Earlier architecture studies use other feature sizes or hidden layers. Their original configurations and source snapshots are supplied; regenerating those inputs from images uses the original feature-preparation scripts and the upstream datasets. The cached inputs are the latest common 8M setup.
|
| 45 |
+
|
| 46 |
+
## Verification
|
| 47 |
+
|
| 48 |
+
`scripts/replay_latest_results.py` recomputes the latest Tables 1 and 3 using the saved per-example outputs. `verification_latest.json` records 80 main runs, 320 Table 3 runs and 19,392 checked curve points. `weight_reload_verification.json` checks all 80 main weights against saved logits using the released cache loader (128 validation inputs per run).
|
| 49 |
+
|
| 50 |
+
Original code is MIT licensed. Derived data retain the licenses of their source datasets; consult `MODELS_AND_DATA.md` before redistribution.
|
releases/20260913_lr/MODELS_AND_DATA.md
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Upstream models and datasets
|
| 2 |
+
|
| 3 |
+
### Vision-language models and router teacher
|
| 4 |
+
|
| 5 |
+
| Model | Hugging Face download | Role |
|
| 6 |
+
|---|---|---|
|
| 7 |
+
| MiniCPM-V-4.5 | [openbmb/MiniCPM-V-4_5](https://huggingface.co/openbmb/MiniCPM-V-4_5) | VLM in the main comparisons |
|
| 8 |
+
| Qwen3.8-27B-FP8 | [Qwen/Qwen3.8-27B-FP8](https://huggingface.co/Qwen/Qwen3.8-27B-FP8) | FP8 VLM in the main comparisons |
|
| 9 |
+
| SigLIP2 Base, patch 16, resolution 224 | [google/siglip2-base-patch16-224](https://huggingface.co/google/siglip2-base-patch16-224) | Teacher for the compact image/text encoders |
|
| 10 |
+
|
| 11 |
+
```bash
|
| 12 |
+
hf download openbmb/MiniCPM-V-4_5 --local-dir models/MiniCPM-V-4_5
|
| 13 |
+
hf download Qwen/Qwen3.8-27B-FP8 --local-dir models/Qwen3.8-27B-FP8
|
| 14 |
+
hf download google/siglip2-base-patch16-224 --local-dir models/siglip2-base-patch16-224
|
| 15 |
+
```
|
| 16 |
+
|
| 17 |
+
Follow each model card for serving dependencies. Preserve the Qwen FP8 variant and the task prompts in [`prompts/`](prompts/) to match the experiments.
|
| 18 |
+
|
| 19 |
+
### Specialist models
|
| 20 |
+
|
| 21 |
+
| Specialist / task | Download or implementation | Checkpoint note |
|
| 22 |
+
|---|---|---|
|
| 23 |
+
| InstanceVG / gRefCOCO | [Hugging Face weights](https://huggingface.co/Dmmm997/InstanceVG/tree/main/gres/InstanceVG-grefcoco), [implementation](https://github.com/Dmmm1997/InstanceVG) | Use the gRefCOCO checkpoint |
|
| 24 |
+
| ViLT / NLVR2 | [dandelin/vilt-b32-finetuned-nlvr2](https://huggingface.co/dandelin/vilt-b32-finetuned-nlvr2) | Task-finetuned upstream weights |
|
| 25 |
+
| BEiT-3 / NLVR2 | [Raghavan/beit3_base_patch16_224_nlvr2](https://huggingface.co/Raghavan/beit3_base_patch16_224_nlvr2), [implementation](https://github.com/microsoft/unilm/tree/master/beit3) | Task-finetuned upstream weights; preserve the recorded input processing |
|
| 26 |
+
| Grounding DINO / gRefCOCO | [Implementation and checkpoints](https://github.com/IDEA-Research/GroundingDINO) | Match the experiment's architecture and checkpoint |
|
| 27 |
+
| ResNet-50 / CUB-200-2011 | [Torchvision ResNet-50](https://docs.pytorch.org/vision/stable/models/generated/torchvision.models.resnet50.html) | Initialization only; the CUB-trained checkpoint is also required |
|
| 28 |
+
| GLSim / CUB-200-2011 | Paper checkpoint: `cub_vit_b16_16_2.pth` | Exact public download URL pending |
|
| 29 |
+
| YOLO26x / ConstructionSite | [Our trained weights](https://huggingface.co/nohi191212/ModelCollaboration/resolve/main/weights.tar) | `specialists/yolo26x/best.pt` inside the archive |
|
| 30 |
+
| RT-DETR-X / ConstructionSite | [Our trained weights](https://huggingface.co/nohi191212/ModelCollaboration/resolve/main/weights.tar) | `specialists/rtdetr_x_fp32/best.pt`; recorded inference uses FP32 |
|
| 31 |
+
|
| 32 |
+
### Datasets
|
| 33 |
+
|
| 34 |
+
| Dataset | Official download / access instructions | Required contents |
|
| 35 |
+
|---|---|---|
|
| 36 |
+
| CUB-200-2011 | [Caltech dataset record](https://data.caltech.edu/records/65de6-vp158) | Images, classes, bounding boxes and original split annotations |
|
| 37 |
+
| gRefCOCO | [Official annotations](https://github.com/henghuiding/gRefCOCO), [COCO images](https://cocodataset.org/#download) | Expressions, annotations and corresponding COCO images; follow upstream image-version instructions |
|
| 38 |
+
| NLVR2 | [Official dataset page](https://lil.nlp.cornell.edu/nlvr/), [download instructions](https://github.com/lil-lab/nlvr/tree/master/nlvr2) | Paired images, statements and labels |
|
| 39 |
+
| ConstructionSite (ConstructionSite10k) | [LouisChen15/ConstructionSite](https://huggingface.co/datasets/LouisChen15/ConstructionSite) | Images and annotations; accept the dataset's access conditions |
|
| 40 |
+
|
| 41 |
+
For ConstructionSite, after access is granted:
|
| 42 |
+
|
| 43 |
+
```bash
|
| 44 |
+
hf auth login
|
| 45 |
+
hf download LouisChen15/ConstructionSite --repo-type dataset --local-dir data/constructionsite10k
|
| 46 |
+
```
|
| 47 |
+
|
| 48 |
+
Dataset and model licenses remain those of their authors. ConstructionSite is distributed under CC BY-NC 4.0. Original dataset splits are not automatically the router train/validation/test splits: follow the split definitions, exclusions and evaluation rules in [`paper/supplementary.pdf`](paper/supplementary.pdf).
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
Upstream models and datasets retain their original licenses.
|
releases/20260913_lr/REPRODUCE.md
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Latest release: 20260913_lr
|
| 2 |
+
|
| 3 |
+
The release contains the latest paper results and the full experiment record. Download artifacts from `nohi191212/ModelCollaboration` on Hugging Face under `releases/20260913_lr/`.
|
| 4 |
+
|
| 5 |
+
## Weights and records
|
| 6 |
+
|
| 7 |
+
`full_budget_20260913_weights.tar.gz` contains exactly **80 router checkpoints**: 16 model pairs × five seeds for the main table. This release publishes no other router weights.
|
| 8 |
+
|
| 9 |
+
The study-specific `_records.tar.gz` archives retain configurations, training histories, validation/test outputs, curves and selection records for the experiments. Extract them into the same artifact directory; paths under `outputs/` are preserved. Earlier fixed-answer CSS/OvA adaptations are separate from the corrected K-class implementations.
|
| 10 |
+
|
| 11 |
+
Previously released backbone and specialist weights remain in the root-level `weights.tar`. External models and datasets are linked in `MODELS_AND_DATA.md`.
|
| 12 |
+
|
| 13 |
+
## Data
|
| 14 |
+
|
| 15 |
+
Eight `training_inputs_<expert>.tar.gz` files contain the train/validation/test arrays and example records for the latest 8M frozen-backbone setup. They contain image/text features, the selected specialist hidden state, confidence, output features, validity masks and labels. `input_manifest.json` records dimensions, sample counts and hidden-layer names. Set `ICASSP_DATA_ROOT` to the directory containing `training_inputs/` and this manifest.
|
| 16 |
+
|
| 17 |
+
`paired_outcomes.tar.gz` provides the paired correctness and task-metric sufficient statistics used for result replay. `paper_replay_runs.tar.gz` contains the 400 training runs used by the latest Tables 1 and 3, with raw logits and configurations. These two files are sufficient for `scripts/replay_latest_results.py`.
|
| 18 |
+
|
| 19 |
+
`additional_analysis_records.tar.gz` contains scoring explorations and cost records. `experiment_sources.tar.gz` preserves original experiment source snapshots, also included in the GitHub repository.
|
| 20 |
+
|
| 21 |
+
The latest `paper/data/` directory stores table and figure summaries. Main weights were selected over the full 0–100% validation curve. Table 3 retains its earlier training selection setting, as specified in the supplement and run configurations.
|
| 22 |
+
|
| 23 |
+
## Training examples
|
| 24 |
+
|
| 25 |
+
Install the dependencies in `requirements.txt` and a PyTorch build for your GPU. Commands below are run from the repository root. They start training only when invoked.
|
| 26 |
+
|
| 27 |
+
For normalized LR, download/extract a specialist's input archive and `input_manifest.json`, then run:
|
| 28 |
+
|
| 29 |
+
```bash
|
| 30 |
+
export ICASSP_DATA_ROOT="$PWD/artifacts/releases/20260913_lr"
|
| 31 |
+
python code/experiments/full_budget_train.py --source configs/full_budget/full_cub_MiniCPM_s2026 --output outputs/my_lr_run
|
| 32 |
+
```
|
| 33 |
+
|
| 34 |
+
For post-hoc deferral, extract its records archive, choose a saved `config.json`, and run:
|
| 35 |
+
|
| 36 |
+
```bash
|
| 37 |
+
python code/experiments/train_ltd.py --root "$PWD" --config "$ICASSP_DATA_ROOT/outputs/ltd_fixed_backbone_20260913/runs/cub_MiniCPM_posthoc_c0_s2026/config.json" --output outputs/my_posthoc_run --device cuda
|
| 38 |
+
```
|
| 39 |
+
|
| 40 |
+
Use `train_three.py` for a configuration from `three_state_20260913`, and `train_native.py` for a configuration from `ltd_kclass_native_20260913`. Configurations are supplied in the records and weight archives. The original objective experiment sources are in `experiments/source_snapshots/`; the five-seed objective trainer is also in `code/objectives/`.
|
| 41 |
+
|
| 42 |
+
To evaluate a newly trained post-hoc/three-state/native run, use its corresponding `evaluate_ltd.py`, `evaluate_three.py` or `evaluate_native.py` with `--help` for arguments. Input paths are resolved through `ICASSP_DATA_ROOT`.
|
| 43 |
+
|
| 44 |
+
Earlier architecture studies use other feature sizes or hidden layers. Their original configurations and source snapshots are supplied; regenerating those inputs from images uses the original feature-preparation scripts and the upstream datasets. The cached inputs are the latest common 8M setup.
|
| 45 |
+
|
| 46 |
+
## Verification
|
| 47 |
+
|
| 48 |
+
`scripts/replay_latest_results.py` recomputes the latest Tables 1 and 3 using the saved per-example outputs. `verification_latest.json` records 80 main runs, 320 Table 3 runs and 19,392 checked curve points. `weight_reload_verification.json` checks all 80 main weights against saved logits using the released cache loader (128 validation inputs per run).
|
| 49 |
+
|
| 50 |
+
Original code is MIT licensed. Derived data retain the licenses of their source datasets; consult `MODELS_AND_DATA.md` before redistribution.
|
releases/20260913_lr/appendix.pdf
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b5bfda42c8b67bb5b356d1d316a9f48f9c4466019482dc232eb0017049236119
|
| 3 |
+
size 1090203
|
releases/20260913_lr/main.pdf
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c5a140a24780ff19e84d870579fc094ab7b8c76296e123f1ab64d2eb52feba38
|
| 3 |
+
size 4826693
|
releases/20260913_lr/paired_outcomes.tar.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5bc2d10e03e7124ff1a073120cfc1e7951cbb4d8ef016b5a305c30a4c78143fd
|
| 3 |
+
size 2668397
|
releases/20260913_lr/paper_data/budget_diagnostics.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
[{"key": ["cub", "cub", "MiniCPM"], "areas": {"confidence": {"validation_fixed_A25": 0.9058333333333335, "test_fixed_A25": 0.6472903003106663, "test_ranked_A25_diagnostic": 0.7895926820849155, "selected_validation_actual_calls": 0.0, "selected_test_actual_calls": 0.0}, "learned": {"validation_fixed_A25": 0.9740333333333333, "test_fixed_A25": 0.8063444943044529, "test_ranked_A25_diagnostic": 0.8089368312046945, "selected_validation_actual_calls": 0.004, "selected_test_actual_calls": 0.0059371763893683125}}}, {"key": ["cub", "cub", "Qwen3.8"], "areas": {"confidence": {"validation_fixed_A25": 0.9171666666666667, "test_fixed_A25": 0.7316189161201244, "test_ranked_A25_diagnostic": 0.8101484294097343, "selected_validation_actual_calls": 0.0, "selected_test_actual_calls": 0.0}, "learned": {"validation_fixed_A25": 0.9834833333333335, "test_fixed_A25": 0.8129651363479462, "test_ranked_A25_diagnostic": 0.8117915084570246, "selected_validation_actual_calls": 0.0, "selected_test_actual_calls": 0.0}}}, {"key": ["cub", "glsim", "MiniCPM"], "areas": {"confidence": {"validation_fixed_A25": 0.9005000000000001, "test_fixed_A25": 0.7253365550569554, "test_ranked_A25_diagnostic": 0.8519848118743528, "selected_validation_actual_calls": 0.0, "selected_test_actual_calls": 0.0}, "learned": {"validation_fixed_A25": 0.9781666666666666, "test_fixed_A25": 0.8896064894718676, "test_ranked_A25_diagnostic": 0.8920262340352089, "selected_validation_actual_calls": 0.004, "selected_test_actual_calls": 0.008491542975491888}}}, {"key": ["cub", "glsim", "Qwen3.8"], "areas": {"confidence": {"validation_fixed_A25": 0.9213333333333334, "test_fixed_A25": 0.7860545391784605, "test_ranked_A25_diagnostic": 0.8691232309285468, "selected_validation_actual_calls": 0.0, "selected_test_actual_calls": 0.0}, "learned": {"validation_fixed_A25": 0.9858, "test_fixed_A25": 0.8977666551605109, "test_ranked_A25_diagnostic": 0.897169485674836, "selected_validation_actual_calls": 0.0, "selected_test_actual_calls": 0.0}}}, {"key": ["grefcoco", "groundingdino", "MiniCPM"], "areas": {"confidence": {"validation_fixed_A25": 0.5566729917773561, "test_fixed_A25": 0.27538212857669514, "test_ranked_A25_diagnostic": 0.2786603522105323, "selected_validation_actual_calls": 1.0, "selected_test_actual_calls": 1.0}, "learned": {"validation_fixed_A25": 0.6126923887834703, "test_fixed_A25": 0.37841703768822854, "test_ranked_A25_diagnostic": 0.3486317102912402, "selected_validation_actual_calls": 0.4579520697167756, "selected_test_actual_calls": 0.5818676800045374}}}, {"key": ["grefcoco", "groundingdino", "Qwen3.8"], "areas": {"confidence": {"validation_fixed_A25": 0.5648886077728582, "test_fixed_A25": 0.28625187873975555, "test_ranked_A25_diagnostic": 0.29324220854720245, "selected_validation_actual_calls": 1.0, "selected_test_actual_calls": 1.0}, "learned": {"validation_fixed_A25": 0.647486119895987, "test_fixed_A25": 0.42089272041516607, "test_ranked_A25_diagnostic": 0.3745852593369821, "selected_validation_actual_calls": 0.7759786351816713, "selected_test_actual_calls": 0.8846666477611093}}}, {"key": ["grefcoco", "instancevg", "MiniCPM"], "areas": {"confidence": {"validation_fixed_A25": 0.7465176751704266, "test_fixed_A25": 0.6639253608598249, "test_ranked_A25_diagnostic": 0.6590023537418825, "selected_validation_actual_calls": 0.6499402628434886, "selected_test_actual_calls": 0.5209426310864078}, "learned": {"validation_fixed_A25": 0.7538997821350764, "test_fixed_A25": 0.678765278053484, "test_ranked_A25_diagnostic": 0.6785259336982106, "selected_validation_actual_calls": 0.14395951929158762, "selected_test_actual_calls": 0.19038085245157815}}}, {"key": ["grefcoco", "instancevg", "Qwen3.8"], "areas": {"confidence": {"validation_fixed_A25": 0.7477616136060159, "test_fixed_A25": 0.6640728242066757, "test_ranked_A25_diagnostic": 0.6611405722712191, "selected_validation_actual_calls": 0.7099585353854804, "selected_test_actual_calls": 0.6838612710206166}, "learned": {"validation_fixed_A25": 0.7825750228406776, "test_fixed_A25": 0.7278393783852765, "test_ranked_A25_diagnostic": 0.7134446870657629, "selected_validation_actual_calls": 0.45595614589921996, "selected_test_actual_calls": 0.6200153135014037}}}, {"key": ["nlvr2", "nlvr2", "MiniCPM"], "areas": {"confidence": {"validation_fixed_A25": 0.8284302492122602, "test_fixed_A25": 0.8339025615268711, "test_ranked_A25_diagnostic": 0.835882901628758, "selected_validation_actual_calls": 0.8587797192781438, "selected_test_actual_calls": 0.8580038745784602}, "learned": {"validation_fixed_A25": 0.8353480378115155, "test_fixed_A25": 0.8422214249838561, "test_ranked_A25_diagnostic": 0.8411638085671236, "selected_validation_actual_calls": 0.9139215124606132, "selected_test_actual_calls": 0.9110425486116093}}}, {"key": ["nlvr2", "nlvr2", "Qwen3.8"], "areas": {"confidence": {"validation_fixed_A25": 0.8312804354053279, "test_fixed_A25": 0.8343976465523427, "test_ranked_A25_diagnostic": 0.8361340317141422, "selected_validation_actual_calls": 0.8587797192781438, "selected_test_actual_calls": 0.8580038745784602}, "learned": {"validation_fixed_A25": 0.8401432254368376, "test_fixed_A25": 0.8425816172777499, "test_ranked_A25_diagnostic": 0.8422458204778647, "selected_validation_actual_calls": 0.8419077628186766, "selected_test_actual_calls": 0.8434096290449882}}}, {"key": ["nlvr2", "vilt", "MiniCPM"], "areas": {"confidence": {"validation_fixed_A25": 0.7926095674591808, "test_fixed_A25": 0.7987371744277822, "test_ranked_A25_diagnostic": 0.8005237856066586, "selected_validation_actual_calls": 0.9598968776854769, "selected_test_actual_calls": 0.9593169261677549}, "learned": {"validation_fixed_A25": 0.7940532798625036, "test_fixed_A25": 0.8014909951926528, "test_ranked_A25_diagnostic": 0.8024137188778073, "selected_validation_actual_calls": 0.9819535949584648, "selected_test_actual_calls": 0.9805984071177442}}}, {"key": ["nlvr2", "vilt", "Qwen3.8"], "areas": {"confidence": {"validation_fixed_A25": 0.7940561443712404, "test_fixed_A25": 0.7988735021884195, "test_ranked_A25_diagnostic": 0.8006242376408124, "selected_validation_actual_calls": 0.9598968776854769, "selected_test_actual_calls": 0.9593169261677549}, "learned": {"validation_fixed_A25": 0.7962102549412775, "test_fixed_A25": 0.8036521489560164, "test_ranked_A25_diagnostic": 0.8044127143574658, "selected_validation_actual_calls": 0.9959896877685477, "selected_test_actual_calls": 0.9961684724115664}}}, {"key": ["construction", "yolo26x", "MiniCPM"], "areas": {"confidence": {"validation_fixed_A25": 0.1745164052515915, "test_fixed_A25": 0.17697501725814355, "test_ranked_A25_diagnostic": 0.1716572124279524, "selected_validation_actual_calls": 0.9386590584878745, "selected_test_actual_calls": 0.9254327563249002}, "learned": {"validation_fixed_A25": 0.2280241031947047, "test_fixed_A25": 0.21383535478219448, "test_ranked_A25_diagnostic": 0.21318248319590255, "selected_validation_actual_calls": 0.4573466476462197, "selected_test_actual_calls": 0.4531957390146471}}}, {"key": ["construction", "yolo26x", "Qwen3.8"], "areas": {"confidence": {"validation_fixed_A25": 0.17942553980644182, "test_fixed_A25": 0.18055274894913717, "test_ranked_A25_diagnostic": 0.1793307821359281, "selected_validation_actual_calls": 0.9286733238231099, "selected_test_actual_calls": 0.9221038615179761}, "learned": {"validation_fixed_A25": 0.2325252201898847, "test_fixed_A25": 0.21172533739102792, "test_ranked_A25_diagnostic": 0.21128777332230012, "selected_validation_actual_calls": 0.45135520684736086, "selected_test_actual_calls": 0.44766977363515315}}}, {"key": ["construction", "rtdetr_x_fp32", "MiniCPM"], "areas": {"confidence": {"validation_fixed_A25": 0.08994758708890332, "test_fixed_A25": 0.08325267500178912, "test_ranked_A25_diagnostic": 0.08357421552875581, "selected_validation_actual_calls": 1.0, "selected_test_actual_calls": 1.0}, "learned": {"validation_fixed_A25": 0.10056264714432415, "test_fixed_A25": 0.10905069389769234, "test_ranked_A25_diagnostic": 0.1142703643498825, "selected_validation_actual_calls": 0.7728958630527817, "selected_test_actual_calls": 0.7843541944074567}}}, {"key": ["construction", "rtdetr_x_fp32", "Qwen3.8"], "areas": {"confidence": {"validation_fixed_A25": 0.10053453664183931, "test_fixed_A25": 0.08910633276678805, "test_ranked_A25_diagnostic": 0.08907503728424503, "selected_validation_actual_calls": 1.0, "selected_test_actual_calls": 1.0}, "learned": {"validation_fixed_A25": 0.11153717985565076, "test_fixed_A25": 0.11890339192252425, "test_ranked_A25_diagnostic": 0.12154998969679998, "selected_validation_actual_calls": 0.8308131241084166, "selected_test_actual_calls": 0.8356857523302264}}}]
|
releases/20260913_lr/paper_data/comparisons.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
releases/20260913_lr/paper_data/cost_20260911/construction_router.json
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"component": "router",
|
| 3 |
+
"expert": "yolo26x",
|
| 4 |
+
"task": "construction",
|
| 5 |
+
"endpoint": "Qwen3.8",
|
| 6 |
+
"condition": "both",
|
| 7 |
+
"seed": 42,
|
| 8 |
+
"device": "cuda:0",
|
| 9 |
+
"checkpoint": "/mnt/HithinkOmniSSD/user_workspace/caisihang/research/ICASSP_ModelCollaboration/outputs/router_method_improvement_20260911_r1/conditions/both_yolo26x_Qwen3.8_seed42/best.pt",
|
| 10 |
+
"sample_count": 1024,
|
| 11 |
+
"batch_size": 256,
|
| 12 |
+
"row_count_loaded": 3004,
|
| 13 |
+
"mean_latency_ms_per_sample": 0.004675843752920628,
|
| 14 |
+
"median_latency_ms_per_sample": 0.004634750075638294,
|
| 15 |
+
"latency_ms_per_sample": [
|
| 16 |
+
0.004889749921858311,
|
| 17 |
+
0.004717750009149313,
|
| 18 |
+
0.004544124938547611,
|
| 19 |
+
0.0045517501421272755
|
| 20 |
+
],
|
| 21 |
+
"flops_counted_per_batch": 0,
|
| 22 |
+
"gflops_counted_per_sample": 0.0,
|
| 23 |
+
"operators": {},
|
| 24 |
+
"flops_scope": "GPU FlopCounterMode for the router forward; multiply-add counted as 2 FLOPs.",
|
| 25 |
+
"latency_scope": "A800 CUDA forward latency on cached router features, excluding feature loading and model load.",
|
| 26 |
+
"completed": true
|
| 27 |
+
}
|
releases/20260913_lr/paper_data/cost_20260911/cost_summary.json
ADDED
|
@@ -0,0 +1,1036 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"protocol": {
|
| 4 |
+
"test_results": "C:\\Users\\csh10\\Documents\\CodexProjects\\ICASSP-COLLA\\dse_results\\method_improvement_20260911\\test_results.json",
|
| 5 |
+
"curves_use": "validation-calibrated thresholds transferred unchanged to test",
|
| 6 |
+
"cost_uses": "actual test call fraction",
|
| 7 |
+
"flop_counting": "multiply-add counted as 2 FLOPs; unsupported operations excluded",
|
| 8 |
+
"large_generation_flops": "mean prefill + mean completion tokens * one-token decode probe",
|
| 9 |
+
"seed_aggregation": "mean and population standard deviation across three seeds for learned/original methods"
|
| 10 |
+
},
|
| 11 |
+
"components": {
|
| 12 |
+
"grefcoco|instancevg|MiniCPM": {
|
| 13 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 14 |
+
"task": "grefcoco",
|
| 15 |
+
"expert": "instancevg",
|
| 16 |
+
"endpoint": "MiniCPM",
|
| 17 |
+
"label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 18 |
+
"specialist_gflops_per_input": 7.502680064,
|
| 19 |
+
"specialist_forward_latency_ms": 11.246967077255249,
|
| 20 |
+
"specialist_end_to_end_latency_ms": 16.31702448321448,
|
| 21 |
+
"specialist_sample_count": 64,
|
| 22 |
+
"router_gflops_per_input": 0.000954368,
|
| 23 |
+
"router_flop_count": 954368,
|
| 24 |
+
"router_flop_breakdown": {
|
| 25 |
+
"image": 393728,
|
| 26 |
+
"text": 196608,
|
| 27 |
+
"hidden": 196608,
|
| 28 |
+
"confidence": 2048,
|
| 29 |
+
"output": 512,
|
| 30 |
+
"head_input_to_hidden": 163840,
|
| 31 |
+
"head_hidden_to_classes": 1024
|
| 32 |
+
},
|
| 33 |
+
"router_measured_gflops_per_input": 0.0,
|
| 34 |
+
"router_latency_ms": 0.001830718741985038,
|
| 35 |
+
"router_sample_count": 1024,
|
| 36 |
+
"large_prefill_gflops": 7467.756340960001,
|
| 37 |
+
"large_decode_one_token_gflops": 15.445344384,
|
| 38 |
+
"large_mean_completion_tokens": 169.5,
|
| 39 |
+
"large_completion_token_min": 5,
|
| 40 |
+
"large_completion_token_max": 512,
|
| 41 |
+
"large_estimated_full_generation_gflops": 10085.742214048001,
|
| 42 |
+
"large_batch_wall_latency_ms_per_input": 1447.4320754343353,
|
| 43 |
+
"large_measured_sample_count": 30,
|
| 44 |
+
"large_flops_probe_count": 4,
|
| 45 |
+
"flops_formula": "specialist + router + actual_call_fraction * (prefill + mean_completion_tokens * one_token_decode)",
|
| 46 |
+
"latency_formula": "specialist end-to-end + router + actual_call_fraction * large-model batch-wall latency",
|
| 47 |
+
"specialist_flops_scope": "GPU FlopCounterMode dense products, convolutions and supported attention; multiply-add counted as 2 FLOPs. Elementwise, normalization, custom ops and postprocessing may be missing.",
|
| 48 |
+
"specialist_latency_scope": "A800 wall-clock CUDA forward latency and end-to-end latency including image read/preprocess, excluding model load; batch processing uses the recorded batch size.",
|
| 49 |
+
"large_flops_scope": "HF reference execution on the A800 with real task inputs; dense products/convolutions/supported attention counted with multiply-add=2 FLOPs. vLLM custom kernels, elementwise/normalization/nonlinearities and other unsupported operations are not complete.",
|
| 50 |
+
"large_latency_scope": "A800 vLLM-0.18.0 native generation latency with the frozen task prompt and structured-output schema; model loading and warmup excluded. Batch wall time includes the real request/image preparation path."
|
| 51 |
+
},
|
| 52 |
+
"construction|yolo26x|Qwen3.8": {
|
| 53 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 54 |
+
"task": "construction",
|
| 55 |
+
"expert": "yolo26x",
|
| 56 |
+
"endpoint": "Qwen3.8",
|
| 57 |
+
"label": "Construction / YOLO26x + Qwen3.8",
|
| 58 |
+
"specialist_gflops_per_input": 194.4179712,
|
| 59 |
+
"specialist_forward_latency_ms": 32.08068132400513,
|
| 60 |
+
"specialist_end_to_end_latency_ms": 31.79218670447881,
|
| 61 |
+
"specialist_sample_count": 64,
|
| 62 |
+
"router_gflops_per_input": 0.000726528,
|
| 63 |
+
"router_flop_count": 726528,
|
| 64 |
+
"router_flop_breakdown": {
|
| 65 |
+
"image": 393728,
|
| 66 |
+
"hidden": 196608,
|
| 67 |
+
"confidence": 2048,
|
| 68 |
+
"output": 2048,
|
| 69 |
+
"head_input_to_hidden": 131072,
|
| 70 |
+
"head_hidden_to_classes": 1024
|
| 71 |
+
},
|
| 72 |
+
"router_measured_gflops_per_input": 0.0,
|
| 73 |
+
"router_latency_ms": 0.004675843752920628,
|
| 74 |
+
"router_sample_count": 1024,
|
| 75 |
+
"large_prefill_gflops": 87588.607720992,
|
| 76 |
+
"large_decode_one_token_gflops": 51.905495232,
|
| 77 |
+
"large_mean_completion_tokens": 137.03333333333333,
|
| 78 |
+
"large_completion_token_min": 110,
|
| 79 |
+
"large_completion_token_max": 226,
|
| 80 |
+
"large_estimated_full_generation_gflops": 94701.3907509504,
|
| 81 |
+
"large_batch_wall_latency_ms_per_input": 2599.870914062194,
|
| 82 |
+
"large_measured_sample_count": 30,
|
| 83 |
+
"large_flops_probe_count": 4,
|
| 84 |
+
"flops_formula": "specialist + router + actual_call_fraction * (prefill + mean_completion_tokens * one_token_decode)",
|
| 85 |
+
"latency_formula": "specialist end-to-end + router + actual_call_fraction * large-model batch-wall latency",
|
| 86 |
+
"specialist_flops_scope": "GPU FlopCounterMode around the real Ultralytics predict path; multiply-add counted as 2 FLOPs. Elementwise, postprocessing and unsupported custom ops may be missing.",
|
| 87 |
+
"specialist_latency_scope": "A800 wall-clock latency for real Ultralytics prediction including image loading/preprocess, forward and result postprocessing, excluding model load; batch processing uses the recorded batch size.",
|
| 88 |
+
"large_flops_scope": "HF reference execution on the A800 with real task inputs; dense products/convolutions/supported attention counted with multiply-add=2 FLOPs. vLLM custom kernels, elementwise/normalization/nonlinearities and other unsupported operations are not complete.",
|
| 89 |
+
"large_latency_scope": "A800 vLLM-0.18.0 native generation latency with the frozen task prompt and structured-output schema; model loading and warmup excluded. Batch wall time includes the real request/image preparation path."
|
| 90 |
+
}
|
| 91 |
+
},
|
| 92 |
+
"selected_points": [
|
| 93 |
+
{
|
| 94 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 95 |
+
"pair_label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 96 |
+
"task": "grefcoco",
|
| 97 |
+
"method": "both",
|
| 98 |
+
"method_label": "两项同时改",
|
| 99 |
+
"seed_count": 3,
|
| 100 |
+
"selected_nominal_budget_mean": 0.04666666666666667,
|
| 101 |
+
"selected_nominal_budgets": [
|
| 102 |
+
0.05,
|
| 103 |
+
0.04,
|
| 104 |
+
0.05
|
| 105 |
+
],
|
| 106 |
+
"test_metric": 0.6805338929378291,
|
| 107 |
+
"test_metric_std": 0.0004176374109666599,
|
| 108 |
+
"actual_test_call_fraction": 0.11292289368459858,
|
| 109 |
+
"actual_test_call_fraction_std": 0.012834207132185534,
|
| 110 |
+
"flops_gflops_per_input": 1146.4148301992102,
|
| 111 |
+
"latency_ms_per_input": 179.7670735719058
|
| 112 |
+
},
|
| 113 |
+
{
|
| 114 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 115 |
+
"pair_label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 116 |
+
"task": "grefcoco",
|
| 117 |
+
"method": "original",
|
| 118 |
+
"method_label": "原方法",
|
| 119 |
+
"seed_count": 3,
|
| 120 |
+
"selected_nominal_budget_mean": 0.19333333333333333,
|
| 121 |
+
"selected_nominal_budgets": [
|
| 122 |
+
0.16,
|
| 123 |
+
0.2,
|
| 124 |
+
0.22
|
| 125 |
+
],
|
| 126 |
+
"test_metric": 0.6732741589390202,
|
| 127 |
+
"test_metric_std": 0.0014792314998359972,
|
| 128 |
+
"actual_test_call_fraction": 0.26045240998591535,
|
| 129 |
+
"actual_test_call_fraction_std": 0.019638624816705554,
|
| 130 |
+
"flops_gflops_per_input": 2634.3595005774837,
|
| 131 |
+
"latency_ms_per_input": 393.3060275397443
|
| 132 |
+
},
|
| 133 |
+
{
|
| 134 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 135 |
+
"pair_label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 136 |
+
"task": "grefcoco",
|
| 137 |
+
"method": "confidence",
|
| 138 |
+
"method_label": "置信度路由",
|
| 139 |
+
"seed_count": 1,
|
| 140 |
+
"selected_nominal_budget_mean": 0.65,
|
| 141 |
+
"selected_nominal_budgets": [
|
| 142 |
+
0.65
|
| 143 |
+
],
|
| 144 |
+
"test_metric": 0.6187788900547316,
|
| 145 |
+
"test_metric_std": 0.0,
|
| 146 |
+
"actual_test_call_fraction": 0.5209426310864078,
|
| 147 |
+
"actual_test_call_fraction_std": 0.0,
|
| 148 |
+
"flops_gflops_per_input": 5261.596719877418,
|
| 149 |
+
"latency_ms_per_input": 770.347928897579
|
| 150 |
+
},
|
| 151 |
+
{
|
| 152 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 153 |
+
"pair_label": "Construction / YOLO26x + Qwen3.8",
|
| 154 |
+
"task": "construction",
|
| 155 |
+
"method": "both",
|
| 156 |
+
"method_label": "两项同时改",
|
| 157 |
+
"seed_count": 3,
|
| 158 |
+
"selected_nominal_budget_mean": 0.44333333333333336,
|
| 159 |
+
"selected_nominal_budgets": [
|
| 160 |
+
0.44,
|
| 161 |
+
0.44,
|
| 162 |
+
0.45
|
| 163 |
+
],
|
| 164 |
+
"test_metric": 0.3010820421930382,
|
| 165 |
+
"test_metric_std": 0.001446335574639307,
|
| 166 |
+
"actual_test_call_fraction": 0.4414114513981358,
|
| 167 |
+
"actual_test_call_fraction_std": 0.005785002395799842,
|
| 168 |
+
"flops_gflops_per_input": 41996.69703852701,
|
| 169 |
+
"latency_ms_per_input": 1179.4096561722229
|
| 170 |
+
},
|
| 171 |
+
{
|
| 172 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 173 |
+
"pair_label": "Construction / YOLO26x + Qwen3.8",
|
| 174 |
+
"task": "construction",
|
| 175 |
+
"method": "original",
|
| 176 |
+
"method_label": "原方法",
|
| 177 |
+
"seed_count": 3,
|
| 178 |
+
"selected_nominal_budget_mean": 0.47333333333333333,
|
| 179 |
+
"selected_nominal_budgets": [
|
| 180 |
+
0.5,
|
| 181 |
+
0.46,
|
| 182 |
+
0.46
|
| 183 |
+
],
|
| 184 |
+
"test_metric": 0.3000434768989391,
|
| 185 |
+
"test_metric_std": 0.002173546570267329,
|
| 186 |
+
"actual_test_call_fraction": 0.46671105193075896,
|
| 187 |
+
"actual_test_call_fraction_std": 0.01718389127843443,
|
| 188 |
+
"flops_gflops_per_input": 44392.60439440991,
|
| 189 |
+
"latency_ms_per_input": 1245.185351734382
|
| 190 |
+
},
|
| 191 |
+
{
|
| 192 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 193 |
+
"pair_label": "Construction / YOLO26x + Qwen3.8",
|
| 194 |
+
"task": "construction",
|
| 195 |
+
"method": "confidence",
|
| 196 |
+
"method_label": "置信度路由",
|
| 197 |
+
"seed_count": 1,
|
| 198 |
+
"selected_nominal_budget_mean": 0.93,
|
| 199 |
+
"selected_nominal_budgets": [
|
| 200 |
+
0.93
|
| 201 |
+
],
|
| 202 |
+
"test_metric": 0.2597490004762105,
|
| 203 |
+
"test_metric_std": 0.0,
|
| 204 |
+
"actual_test_call_fraction": 0.9221038615179761,
|
| 205 |
+
"actual_test_call_fraction_std": 0.0,
|
| 206 |
+
"flops_gflops_per_input": 87518.93680030212,
|
| 207 |
+
"latency_ms_per_input": 2429.147871853251
|
| 208 |
+
}
|
| 209 |
+
],
|
| 210 |
+
"fixed_budget_points": [
|
| 211 |
+
{
|
| 212 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 213 |
+
"pair_label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 214 |
+
"task": "grefcoco",
|
| 215 |
+
"method": "both",
|
| 216 |
+
"method_label": "两项同时改",
|
| 217 |
+
"nominal_budget": 0.0,
|
| 218 |
+
"seed_count": 3,
|
| 219 |
+
"test_metric": 0.6695119530385957,
|
| 220 |
+
"test_metric_std": 0.0,
|
| 221 |
+
"actual_test_call_fraction": 0.0,
|
| 222 |
+
"actual_test_call_fraction_std": 0.0,
|
| 223 |
+
"flops_gflops_per_input": 7.503634432,
|
| 224 |
+
"latency_ms_per_input": 16.318855201956467
|
| 225 |
+
},
|
| 226 |
+
{
|
| 227 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 228 |
+
"pair_label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 229 |
+
"task": "grefcoco",
|
| 230 |
+
"method": "both",
|
| 231 |
+
"method_label": "两项同时改",
|
| 232 |
+
"nominal_budget": 0.01,
|
| 233 |
+
"seed_count": 3,
|
| 234 |
+
"test_metric": 0.6743328701471798,
|
| 235 |
+
"test_metric_std": 0.0002504540415259137,
|
| 236 |
+
"actual_test_call_fraction": 0.021268751949635595,
|
| 237 |
+
"actual_test_call_fraction_std": 0.0024468276041848553,
|
| 238 |
+
"flops_gflops_per_input": 222.01478381055546,
|
| 239 |
+
"latency_ms_per_input": 47.10392897831558
|
| 240 |
+
},
|
| 241 |
+
{
|
| 242 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 243 |
+
"pair_label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 244 |
+
"task": "grefcoco",
|
| 245 |
+
"method": "both",
|
| 246 |
+
"method_label": "两项同时改",
|
| 247 |
+
"nominal_budget": 0.05,
|
| 248 |
+
"seed_count": 3,
|
| 249 |
+
"test_metric": 0.6805811568310505,
|
| 250 |
+
"test_metric_std": 0.0004271568320976906,
|
| 251 |
+
"actual_test_call_fraction": 0.11976670542305912,
|
| 252 |
+
"actual_test_call_fraction_std": 0.0032185843726325893,
|
| 253 |
+
"flops_gflops_per_input": 1215.4397511547988,
|
| 254 |
+
"latency_ms_per_input": 189.67302620038757
|
| 255 |
+
},
|
| 256 |
+
{
|
| 257 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 258 |
+
"pair_label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 259 |
+
"task": "grefcoco",
|
| 260 |
+
"method": "both",
|
| 261 |
+
"method_label": "两项同时改",
|
| 262 |
+
"nominal_budget": 0.1,
|
| 263 |
+
"seed_count": 3,
|
| 264 |
+
"test_metric": 0.6804866290446077,
|
| 265 |
+
"test_metric_std": 0.0005385595679416328,
|
| 266 |
+
"actual_test_call_fraction": 0.172031118547297,
|
| 267 |
+
"actual_test_call_fraction_std": 0.0033259813564282133,
|
| 268 |
+
"flops_gflops_per_input": 1742.5651488943693,
|
| 269 |
+
"latency_ms_per_input": 265.32221416016074
|
| 270 |
+
},
|
| 271 |
+
{
|
| 272 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 273 |
+
"pair_label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 274 |
+
"task": "grefcoco",
|
| 275 |
+
"method": "both",
|
| 276 |
+
"method_label": "两项同时改",
|
| 277 |
+
"nominal_budget": 0.25,
|
| 278 |
+
"seed_count": 3,
|
| 279 |
+
"test_metric": 0.6794184650578038,
|
| 280 |
+
"test_metric_std": 0.00038210745789084077,
|
| 281 |
+
"actual_test_call_fraction": 0.24530905859777485,
|
| 282 |
+
"actual_test_call_fraction_std": 0.009454413833556307,
|
| 283 |
+
"flops_gflops_per_input": 2481.627562219953,
|
| 284 |
+
"latency_ms_per_input": 371.38705501097667
|
| 285 |
+
},
|
| 286 |
+
{
|
| 287 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 288 |
+
"pair_label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 289 |
+
"task": "grefcoco",
|
| 290 |
+
"method": "both",
|
| 291 |
+
"method_label": "两项同时改",
|
| 292 |
+
"nominal_budget": 0.5,
|
| 293 |
+
"seed_count": 3,
|
| 294 |
+
"test_metric": 0.664568149807636,
|
| 295 |
+
"test_metric_std": 0.0016062493473305785,
|
| 296 |
+
"actual_test_call_fraction": 0.4310372534006371,
|
| 297 |
+
"actual_test_call_fraction_std": 0.0088737075287505,
|
| 298 |
+
"flops_gflops_per_input": 4354.834256882111,
|
| 299 |
+
"latency_ms_per_input": 640.2160014811561
|
| 300 |
+
},
|
| 301 |
+
{
|
| 302 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 303 |
+
"pair_label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 304 |
+
"task": "grefcoco",
|
| 305 |
+
"method": "both",
|
| 306 |
+
"method_label": "两项同时改",
|
| 307 |
+
"nominal_budget": 0.75,
|
| 308 |
+
"seed_count": 3,
|
| 309 |
+
"test_metric": 0.6150355897115957,
|
| 310 |
+
"test_metric_std": 0.0013848240058531283,
|
| 311 |
+
"actual_test_call_fraction": 0.7057350008034863,
|
| 312 |
+
"actual_test_call_fraction_std": 0.006014701362190238,
|
| 313 |
+
"flops_gflops_per_input": 7125.364923966921,
|
| 314 |
+
"latency_ms_per_input": 1037.8223321215987
|
| 315 |
+
},
|
| 316 |
+
{
|
| 317 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 318 |
+
"pair_label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 319 |
+
"task": "grefcoco",
|
| 320 |
+
"method": "both",
|
| 321 |
+
"method_label": "两项同时改",
|
| 322 |
+
"nominal_budget": 0.99,
|
| 323 |
+
"seed_count": 3,
|
| 324 |
+
"test_metric": 0.5069147075782926,
|
| 325 |
+
"test_metric_std": 9.357773432602366e-05,
|
| 326 |
+
"actual_test_call_fraction": 0.988051687793627,
|
| 327 |
+
"actual_test_call_fraction_std": 0.00048144255369636903,
|
| 328 |
+
"flops_gflops_per_input": 9972.73825167356,
|
| 329 |
+
"latency_ms_per_input": 1446.4565603014837
|
| 330 |
+
},
|
| 331 |
+
{
|
| 332 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 333 |
+
"pair_label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 334 |
+
"task": "grefcoco",
|
| 335 |
+
"method": "both",
|
| 336 |
+
"method_label": "两项同时改",
|
| 337 |
+
"nominal_budget": 1.0,
|
| 338 |
+
"seed_count": 3,
|
| 339 |
+
"test_metric": 0.5000708958398321,
|
| 340 |
+
"test_metric_std": 0.0,
|
| 341 |
+
"actual_test_call_fraction": 1.0,
|
| 342 |
+
"actual_test_call_fraction_std": 0.0,
|
| 343 |
+
"flops_gflops_per_input": 10093.24584848,
|
| 344 |
+
"latency_ms_per_input": 1463.7509306362917
|
| 345 |
+
},
|
| 346 |
+
{
|
| 347 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 348 |
+
"pair_label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 349 |
+
"task": "grefcoco",
|
| 350 |
+
"method": "original",
|
| 351 |
+
"method_label": "原方法",
|
| 352 |
+
"nominal_budget": 0.0,
|
| 353 |
+
"seed_count": 3,
|
| 354 |
+
"test_metric": 0.6695119530385957,
|
| 355 |
+
"test_metric_std": 0.0,
|
| 356 |
+
"actual_test_call_fraction": 0.0,
|
| 357 |
+
"actual_test_call_fraction_std": 0.0,
|
| 358 |
+
"flops_gflops_per_input": 7.503634432,
|
| 359 |
+
"latency_ms_per_input": 16.318855201956467
|
| 360 |
+
},
|
| 361 |
+
{
|
| 362 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 363 |
+
"pair_label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 364 |
+
"task": "grefcoco",
|
| 365 |
+
"method": "original",
|
| 366 |
+
"method_label": "原方法",
|
| 367 |
+
"nominal_budget": 0.01,
|
| 368 |
+
"seed_count": 3,
|
| 369 |
+
"test_metric": 0.6742194368034484,
|
| 370 |
+
"test_metric_std": 0.0005415378002401663,
|
| 371 |
+
"actual_test_call_fraction": 0.04492905689627466,
|
| 372 |
+
"actual_test_call_fraction_std": 0.0027758676387843045,
|
| 373 |
+
"flops_gflops_per_input": 460.64652020812184,
|
| 374 |
+
"latency_ms_per_input": 81.35061327263864
|
| 375 |
+
},
|
| 376 |
+
{
|
| 377 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 378 |
+
"pair_label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 379 |
+
"task": "grefcoco",
|
| 380 |
+
"method": "original",
|
| 381 |
+
"method_label": "原方法",
|
| 382 |
+
"nominal_budget": 0.05,
|
| 383 |
+
"seed_count": 3,
|
| 384 |
+
"test_metric": 0.679891103990018,
|
| 385 |
+
"test_metric_std": 0.0007177890142290943,
|
| 386 |
+
"actual_test_call_fraction": 0.10489748461560276,
|
| 387 |
+
"actual_test_call_fraction_std": 0.007840998690258661,
|
| 388 |
+
"flops_gflops_per_input": 1065.4726231670354,
|
| 389 |
+
"latency_ms_per_input": 168.15083906695963
|
| 390 |
+
},
|
| 391 |
+
{
|
| 392 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 393 |
+
"pair_label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 394 |
+
"task": "grefcoco",
|
| 395 |
+
"method": "original",
|
| 396 |
+
"method_label": "原方法",
|
| 397 |
+
"nominal_budget": 0.1,
|
| 398 |
+
"seed_count": 3,
|
| 399 |
+
"test_metric": 0.680127423456125,
|
| 400 |
+
"test_metric_std": 0.0007907632127485806,
|
| 401 |
+
"actual_test_call_fraction": 0.16706840975904869,
|
| 402 |
+
"actual_test_call_fraction_std": 0.004046760924054268,
|
| 403 |
+
"flops_gflops_per_input": 1692.5125473727062,
|
| 404 |
+
"latency_ms_per_input": 258.1390302790103
|
| 405 |
+
},
|
| 406 |
+
{
|
| 407 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 408 |
+
"pair_label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 409 |
+
"task": "grefcoco",
|
| 410 |
+
"method": "original",
|
| 411 |
+
"method_label": "原方法",
|
| 412 |
+
"nominal_budget": 0.25,
|
| 413 |
+
"seed_count": 3,
|
| 414 |
+
"test_metric": 0.6658726332605469,
|
| 415 |
+
"test_metric_std": 0.0048175982262131,
|
| 416 |
+
"actual_test_call_fraction": 0.3117620924670807,
|
| 417 |
+
"actual_test_call_fraction_std": 0.009206880232216604,
|
| 418 |
+
"flops_gflops_per_input": 3151.855731167172,
|
| 419 |
+
"latency_ms_per_input": 467.5733077433342
|
| 420 |
+
},
|
| 421 |
+
{
|
| 422 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 423 |
+
"pair_label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 424 |
+
"task": "grefcoco",
|
| 425 |
+
"method": "original",
|
| 426 |
+
"method_label": "原方法",
|
| 427 |
+
"nominal_budget": 0.5,
|
| 428 |
+
"seed_count": 3,
|
| 429 |
+
"test_metric": 0.6273242019491629,
|
| 430 |
+
"test_metric_std": 0.009832598413097358,
|
| 431 |
+
"actual_test_call_fraction": 0.5310665570144345,
|
| 432 |
+
"actual_test_call_fraction_std": 0.009248673211785196,
|
| 433 |
+
"flops_gflops_per_input": 5363.704026981612,
|
| 434 |
+
"latency_ms_per_input": 785.0016240151261
|
| 435 |
+
},
|
| 436 |
+
{
|
| 437 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 438 |
+
"pair_label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 439 |
+
"task": "grefcoco",
|
| 440 |
+
"method": "original",
|
| 441 |
+
"method_label": "原方法",
|
| 442 |
+
"nominal_budget": 0.75,
|
| 443 |
+
"seed_count": 3,
|
| 444 |
+
"test_metric": 0.5814404144098159,
|
| 445 |
+
"test_metric_std": 0.00858033252822142,
|
| 446 |
+
"actual_test_call_fraction": 0.7394719677849304,
|
| 447 |
+
"actual_test_call_fraction_std": 0.008502993761525655,
|
| 448 |
+
"flops_gflops_per_input": 7465.627276025616,
|
| 449 |
+
"latency_ms_per_input": 1086.65430025841
|
| 450 |
+
},
|
| 451 |
+
{
|
| 452 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 453 |
+
"pair_label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 454 |
+
"task": "grefcoco",
|
| 455 |
+
"method": "original",
|
| 456 |
+
"method_label": "原方法",
|
| 457 |
+
"nominal_budget": 0.99,
|
| 458 |
+
"seed_count": 3,
|
| 459 |
+
"test_metric": 0.5100152189736173,
|
| 460 |
+
"test_metric_std": 0.0036056620177345906,
|
| 461 |
+
"actual_test_call_fraction": 0.978079006323909,
|
| 462 |
+
"actual_test_call_fraction_std": 0.009338895613008017,
|
| 463 |
+
"flops_gflops_per_input": 9872.15635718717,
|
| 464 |
+
"latency_ms_per_input": 1432.0217812641245
|
| 465 |
+
},
|
| 466 |
+
{
|
| 467 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 468 |
+
"pair_label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 469 |
+
"task": "grefcoco",
|
| 470 |
+
"method": "original",
|
| 471 |
+
"method_label": "原方法",
|
| 472 |
+
"nominal_budget": 1.0,
|
| 473 |
+
"seed_count": 3,
|
| 474 |
+
"test_metric": 0.5000708958398321,
|
| 475 |
+
"test_metric_std": 0.0,
|
| 476 |
+
"actual_test_call_fraction": 1.0,
|
| 477 |
+
"actual_test_call_fraction_std": 0.0,
|
| 478 |
+
"flops_gflops_per_input": 10093.24584848,
|
| 479 |
+
"latency_ms_per_input": 1463.7509306362917
|
| 480 |
+
},
|
| 481 |
+
{
|
| 482 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 483 |
+
"pair_label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 484 |
+
"task": "grefcoco",
|
| 485 |
+
"method": "confidence",
|
| 486 |
+
"method_label": "置信度路由",
|
| 487 |
+
"nominal_budget": 0.0,
|
| 488 |
+
"seed_count": 1,
|
| 489 |
+
"test_metric": 0.6695119530385957,
|
| 490 |
+
"test_metric_std": 0.0,
|
| 491 |
+
"actual_test_call_fraction": 0.0,
|
| 492 |
+
"actual_test_call_fraction_std": 0.0,
|
| 493 |
+
"flops_gflops_per_input": 7.503634432,
|
| 494 |
+
"latency_ms_per_input": 16.318855201956467
|
| 495 |
+
},
|
| 496 |
+
{
|
| 497 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 498 |
+
"pair_label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 499 |
+
"task": "grefcoco",
|
| 500 |
+
"method": "confidence",
|
| 501 |
+
"method_label": "置信度路由",
|
| 502 |
+
"nominal_budget": 0.01,
|
| 503 |
+
"seed_count": 1,
|
| 504 |
+
"test_metric": 0.6686044862887446,
|
| 505 |
+
"test_metric_std": 0.0,
|
| 506 |
+
"actual_test_call_fraction": 0.002467175226157729,
|
| 507 |
+
"actual_test_call_fraction_std": 0.0,
|
| 508 |
+
"flops_gflops_per_input": 32.38692775991243,
|
| 509 |
+
"latency_ms_per_input": 19.889923760014124
|
| 510 |
+
},
|
| 511 |
+
{
|
| 512 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 513 |
+
"pair_label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 514 |
+
"task": "grefcoco",
|
| 515 |
+
"method": "confidence",
|
| 516 |
+
"method_label": "置信度路由",
|
| 517 |
+
"nominal_budget": 0.05,
|
| 518 |
+
"seed_count": 1,
|
| 519 |
+
"test_metric": 0.6662791027422511,
|
| 520 |
+
"test_metric_std": 0.0,
|
| 521 |
+
"actual_test_call_fraction": 0.012534384482318578,
|
| 522 |
+
"actual_test_call_fraction_std": 0.0,
|
| 523 |
+
"flops_gflops_per_input": 133.9222051324287,
|
| 524 |
+
"latency_ms_per_input": 34.46152534749078
|
| 525 |
+
},
|
| 526 |
+
{
|
| 527 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 528 |
+
"pair_label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 529 |
+
"task": "grefcoco",
|
| 530 |
+
"method": "confidence",
|
| 531 |
+
"method_label": "置信度路由",
|
| 532 |
+
"nominal_budget": 0.1,
|
| 533 |
+
"seed_count": 1,
|
| 534 |
+
"test_metric": 0.6645208859144145,
|
| 535 |
+
"test_metric_std": 0.0,
|
| 536 |
+
"actual_test_call_fraction": 0.0345971698380739,
|
| 537 |
+
"actual_test_call_fraction_std": 0.0,
|
| 538 |
+
"flops_gflops_per_input": 356.4417707544502,
|
| 539 |
+
"latency_ms_per_input": 66.39590854483396
|
| 540 |
+
},
|
| 541 |
+
{
|
| 542 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 543 |
+
"pair_label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 544 |
+
"task": "grefcoco",
|
| 545 |
+
"method": "confidence",
|
| 546 |
+
"method_label": "置信度路由",
|
| 547 |
+
"nominal_budget": 0.25,
|
| 548 |
+
"seed_count": 1,
|
| 549 |
+
"test_metric": 0.6596999688058305,
|
| 550 |
+
"test_metric_std": 0.0,
|
| 551 |
+
"actual_test_call_fraction": 0.12477667810452883,
|
| 552 |
+
"actual_test_call_fraction_std": 0.0,
|
| 553 |
+
"flops_gflops_per_input": 1265.9690441195253,
|
| 554 |
+
"latency_ms_per_input": 196.92462135659662
|
| 555 |
+
},
|
| 556 |
+
{
|
| 557 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 558 |
+
"pair_label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 559 |
+
"task": "grefcoco",
|
| 560 |
+
"method": "confidence",
|
| 561 |
+
"method_label": "置信度路由",
|
| 562 |
+
"nominal_budget": 0.5,
|
| 563 |
+
"seed_count": 1,
|
| 564 |
+
"test_metric": 0.6527521765022829,
|
| 565 |
+
"test_metric_std": 0.0,
|
| 566 |
+
"actual_test_call_fraction": 0.2345234381646485,
|
| 567 |
+
"actual_test_call_fraction_std": 0.0,
|
| 568 |
+
"flops_gflops_per_input": 2372.8465749128713,
|
| 569 |
+
"latency_ms_per_input": 355.77560204260965
|
| 570 |
+
},
|
| 571 |
+
{
|
| 572 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 573 |
+
"pair_label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 574 |
+
"task": "grefcoco",
|
| 575 |
+
"method": "confidence",
|
| 576 |
+
"method_label": "置信度路由",
|
| 577 |
+
"nominal_budget": 0.75,
|
| 578 |
+
"seed_count": 1,
|
| 579 |
+
"test_metric": 0.567762243711539,
|
| 580 |
+
"test_metric_std": 0.0,
|
| 581 |
+
"actual_test_call_fraction": 0.7478093185491875,
|
| 582 |
+
"actual_test_call_fraction_std": 0.0,
|
| 583 |
+
"flops_gflops_per_input": 7549.715646582009,
|
| 584 |
+
"latency_ms_per_input": 1098.722049178743
|
| 585 |
+
},
|
| 586 |
+
{
|
| 587 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 588 |
+
"pair_label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 589 |
+
"task": "grefcoco",
|
| 590 |
+
"method": "confidence",
|
| 591 |
+
"method_label": "置信度路由",
|
| 592 |
+
"nominal_budget": 0.99,
|
| 593 |
+
"seed_count": 1,
|
| 594 |
+
"test_metric": 0.5019425460114001,
|
| 595 |
+
"test_metric_std": 0.0,
|
| 596 |
+
"actual_test_call_fraction": 0.9908118991577575,
|
| 597 |
+
"actual_test_call_fraction_std": 0.0,
|
| 598 |
+
"flops_gflops_per_input": 10000.577031948465,
|
| 599 |
+
"latency_ms_per_input": 1450.4517787649047
|
| 600 |
+
},
|
| 601 |
+
{
|
| 602 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 603 |
+
"pair_label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 604 |
+
"task": "grefcoco",
|
| 605 |
+
"method": "confidence",
|
| 606 |
+
"method_label": "置信度路由",
|
| 607 |
+
"nominal_budget": 1.0,
|
| 608 |
+
"seed_count": 1,
|
| 609 |
+
"test_metric": 0.5000708958398321,
|
| 610 |
+
"test_metric_std": 0.0,
|
| 611 |
+
"actual_test_call_fraction": 1.0,
|
| 612 |
+
"actual_test_call_fraction_std": 0.0,
|
| 613 |
+
"flops_gflops_per_input": 10093.24584848,
|
| 614 |
+
"latency_ms_per_input": 1463.7509306362917
|
| 615 |
+
},
|
| 616 |
+
{
|
| 617 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 618 |
+
"pair_label": "Construction / YOLO26x + Qwen3.8",
|
| 619 |
+
"task": "construction",
|
| 620 |
+
"method": "both",
|
| 621 |
+
"method_label": "两项同时改",
|
| 622 |
+
"nominal_budget": 0.0,
|
| 623 |
+
"seed_count": 3,
|
| 624 |
+
"test_metric": 0.18129624718318374,
|
| 625 |
+
"test_metric_std": 2.7755575615628914e-17,
|
| 626 |
+
"actual_test_call_fraction": 0.0,
|
| 627 |
+
"actual_test_call_fraction_std": 0.0,
|
| 628 |
+
"flops_gflops_per_input": 194.418697728,
|
| 629 |
+
"latency_ms_per_input": 31.79686254823173
|
| 630 |
+
},
|
| 631 |
+
{
|
| 632 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 633 |
+
"pair_label": "Construction / YOLO26x + Qwen3.8",
|
| 634 |
+
"task": "construction",
|
| 635 |
+
"method": "both",
|
| 636 |
+
"method_label": "两项同时改",
|
| 637 |
+
"nominal_budget": 0.01,
|
| 638 |
+
"seed_count": 3,
|
| 639 |
+
"test_metric": 0.1834834493716783,
|
| 640 |
+
"test_metric_std": 0.0012311365629078506,
|
| 641 |
+
"actual_test_call_fraction": 0.01187305814469596,
|
| 642 |
+
"actual_test_call_fraction_std": 0.0025254786233335643,
|
| 643 |
+
"flops_gflops_per_input": 1318.8138164976065,
|
| 644 |
+
"latency_ms_per_input": 62.665281079596
|
| 645 |
+
},
|
| 646 |
+
{
|
| 647 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 648 |
+
"pair_label": "Construction / YOLO26x + Qwen3.8",
|
| 649 |
+
"task": "construction",
|
| 650 |
+
"method": "both",
|
| 651 |
+
"method_label": "两项同时改",
|
| 652 |
+
"nominal_budget": 0.05,
|
| 653 |
+
"seed_count": 3,
|
| 654 |
+
"test_metric": 0.19244312207391248,
|
| 655 |
+
"test_metric_std": 0.0014149660525805208,
|
| 656 |
+
"actual_test_call_fraction": 0.04804704837993786,
|
| 657 |
+
"actual_test_call_fraction_std": 0.0025254786233335656,
|
| 658 |
+
"flops_gflops_per_input": 4744.541000786314,
|
| 659 |
+
"latency_ms_per_input": 156.71298613777122
|
| 660 |
+
},
|
| 661 |
+
{
|
| 662 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 663 |
+
"pair_label": "Construction / YOLO26x + Qwen3.8",
|
| 664 |
+
"task": "construction",
|
| 665 |
+
"method": "both",
|
| 666 |
+
"method_label": "两项同时改",
|
| 667 |
+
"nominal_budget": 0.1,
|
| 668 |
+
"seed_count": 3,
|
| 669 |
+
"test_metric": 0.2058996264309502,
|
| 670 |
+
"test_metric_std": 0.000756529123691676,
|
| 671 |
+
"actual_test_call_fraction": 0.10707944962272525,
|
| 672 |
+
"actual_test_call_fraction_std": 0.0037002512207130658,
|
| 673 |
+
"flops_gflops_per_input": 10334.991497846413,
|
| 674 |
+
"latency_ms_per_input": 310.1896091161431
|
| 675 |
+
},
|
| 676 |
+
{
|
| 677 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 678 |
+
"pair_label": "Construction / YOLO26x + Qwen3.8",
|
| 679 |
+
"task": "construction",
|
| 680 |
+
"method": "both",
|
| 681 |
+
"method_label": "两项同时改",
|
| 682 |
+
"nominal_budget": 0.25,
|
| 683 |
+
"seed_count": 3,
|
| 684 |
+
"test_metric": 0.25617136079673564,
|
| 685 |
+
"test_metric_std": 0.0020077364088740926,
|
| 686 |
+
"actual_test_call_fraction": 0.26498002663115844,
|
| 687 |
+
"actual_test_call_fraction_std": 0.008293336414138841,
|
| 688 |
+
"flops_gflops_per_input": 25288.39574092258,
|
| 689 |
+
"latency_ms_per_input": 720.7107265940061
|
| 690 |
+
},
|
| 691 |
+
{
|
| 692 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 693 |
+
"pair_label": "Construction / YOLO26x + Qwen3.8",
|
| 694 |
+
"task": "construction",
|
| 695 |
+
"method": "both",
|
| 696 |
+
"method_label": "两项同时改",
|
| 697 |
+
"nominal_budget": 0.5,
|
| 698 |
+
"seed_count": 3,
|
| 699 |
+
"test_metric": 0.29980106360416753,
|
| 700 |
+
"test_metric_std": 0.002221016752728669,
|
| 701 |
+
"actual_test_call_fraction": 0.49511762094984463,
|
| 702 |
+
"actual_test_call_fraction_std": 0.00440231567351916,
|
| 703 |
+
"flops_gflops_per_input": 47082.74598698018,
|
| 704 |
+
"latency_ms_per_input": 1319.038764295403
|
| 705 |
+
},
|
| 706 |
+
{
|
| 707 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 708 |
+
"pair_label": "Construction / YOLO26x + Qwen3.8",
|
| 709 |
+
"task": "construction",
|
| 710 |
+
"method": "both",
|
| 711 |
+
"method_label": "两项同时改",
|
| 712 |
+
"nominal_budget": 0.75,
|
| 713 |
+
"seed_count": 3,
|
| 714 |
+
"test_metric": 0.28788636673929996,
|
| 715 |
+
"test_metric_std": 0.0011284209510194398,
|
| 716 |
+
"actual_test_call_fraction": 0.7496671105193076,
|
| 717 |
+
"actual_test_call_fraction_std": 0.005429263125932131,
|
| 718 |
+
"flops_gflops_per_input": 71188.93666415288,
|
| 719 |
+
"latency_ms_per_input": 1980.8345784164278
|
| 720 |
+
},
|
| 721 |
+
{
|
| 722 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 723 |
+
"pair_label": "Construction / YOLO26x + Qwen3.8",
|
| 724 |
+
"task": "construction",
|
| 725 |
+
"method": "both",
|
| 726 |
+
"method_label": "两项同时改",
|
| 727 |
+
"nominal_budget": 0.99,
|
| 728 |
+
"seed_count": 3,
|
| 729 |
+
"test_metric": 0.27239176103160984,
|
| 730 |
+
"test_metric_std": 0.0005578690368974412,
|
| 731 |
+
"actual_test_call_fraction": 0.9862405681313805,
|
| 732 |
+
"actual_test_call_fraction_std": 0.0023116585281845767,
|
| 733 |
+
"flops_gflops_per_input": 93592.7721147772,
|
| 734 |
+
"latency_ms_per_input": 2595.8950299011817
|
| 735 |
+
},
|
| 736 |
+
{
|
| 737 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 738 |
+
"pair_label": "Construction / YOLO26x + Qwen3.8",
|
| 739 |
+
"task": "construction",
|
| 740 |
+
"method": "both",
|
| 741 |
+
"method_label": "两项同时改",
|
| 742 |
+
"nominal_budget": 1.0,
|
| 743 |
+
"seed_count": 3,
|
| 744 |
+
"test_metric": 0.27001009067886245,
|
| 745 |
+
"test_metric_std": 0.0,
|
| 746 |
+
"actual_test_call_fraction": 1.0,
|
| 747 |
+
"actual_test_call_fraction_std": 0.0,
|
| 748 |
+
"flops_gflops_per_input": 94895.80944867841,
|
| 749 |
+
"latency_ms_per_input": 2631.667776610426
|
| 750 |
+
},
|
| 751 |
+
{
|
| 752 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 753 |
+
"pair_label": "Construction / YOLO26x + Qwen3.8",
|
| 754 |
+
"task": "construction",
|
| 755 |
+
"method": "original",
|
| 756 |
+
"method_label": "原方法",
|
| 757 |
+
"nominal_budget": 0.0,
|
| 758 |
+
"seed_count": 3,
|
| 759 |
+
"test_metric": 0.18129624718318374,
|
| 760 |
+
"test_metric_std": 2.7755575615628914e-17,
|
| 761 |
+
"actual_test_call_fraction": 0.0,
|
| 762 |
+
"actual_test_call_fraction_std": 0.0,
|
| 763 |
+
"flops_gflops_per_input": 194.418697728,
|
| 764 |
+
"latency_ms_per_input": 31.79686254823173
|
| 765 |
+
},
|
| 766 |
+
{
|
| 767 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 768 |
+
"pair_label": "Construction / YOLO26x + Qwen3.8",
|
| 769 |
+
"task": "construction",
|
| 770 |
+
"method": "original",
|
| 771 |
+
"method_label": "原方法",
|
| 772 |
+
"nominal_budget": 0.01,
|
| 773 |
+
"seed_count": 3,
|
| 774 |
+
"test_metric": 0.18300903853620068,
|
| 775 |
+
"test_metric_std": 0.000992883110506064,
|
| 776 |
+
"actual_test_call_fraction": 0.010319573901464712,
|
| 777 |
+
"actual_test_call_fraction_std": 0.0032954377285658006,
|
| 778 |
+
"flops_gflops_per_input": 1171.6966981539194,
|
| 779 |
+
"latency_ms_per_input": 58.626422580165155
|
| 780 |
+
},
|
| 781 |
+
{
|
| 782 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 783 |
+
"pair_label": "Construction / YOLO26x + Qwen3.8",
|
| 784 |
+
"task": "construction",
|
| 785 |
+
"method": "original",
|
| 786 |
+
"method_label": "原方法",
|
| 787 |
+
"nominal_budget": 0.05,
|
| 788 |
+
"seed_count": 3,
|
| 789 |
+
"test_metric": 0.19493527207894204,
|
| 790 |
+
"test_metric_std": 0.0032238119689520237,
|
| 791 |
+
"actual_test_call_fraction": 0.049822458943630715,
|
| 792 |
+
"actual_test_call_fraction_std": 0.011397380483895993,
|
| 793 |
+
"flops_gflops_per_input": 4912.674850321956,
|
| 794 |
+
"latency_ms_per_input": 161.32882442283505
|
| 795 |
+
},
|
| 796 |
+
{
|
| 797 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 798 |
+
"pair_label": "Construction / YOLO26x + Qwen3.8",
|
| 799 |
+
"task": "construction",
|
| 800 |
+
"method": "original",
|
| 801 |
+
"method_label": "原方法",
|
| 802 |
+
"nominal_budget": 0.1,
|
| 803 |
+
"seed_count": 3,
|
| 804 |
+
"test_metric": 0.20346296196880112,
|
| 805 |
+
"test_metric_std": 0.004110692938093332,
|
| 806 |
+
"actual_test_call_fraction": 0.09576120727918332,
|
| 807 |
+
"actual_test_call_fraction_std": 0.006259345218007303,
|
| 808 |
+
"flops_gflops_per_input": 9263.138207056696,
|
| 809 |
+
"latency_ms_per_input": 280.76364004886125
|
| 810 |
+
},
|
| 811 |
+
{
|
| 812 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 813 |
+
"pair_label": "Construction / YOLO26x + Qwen3.8",
|
| 814 |
+
"task": "construction",
|
| 815 |
+
"method": "original",
|
| 816 |
+
"method_label": "原方法",
|
| 817 |
+
"nominal_budget": 0.25,
|
| 818 |
+
"seed_count": 3,
|
| 819 |
+
"test_metric": 0.24461408052105227,
|
| 820 |
+
"test_metric_std": 0.0022654085644492608,
|
| 821 |
+
"actual_test_call_fraction": 0.2533288948069241,
|
| 822 |
+
"actual_test_call_fraction_std": 0.0017823322685540256,
|
| 823 |
+
"flops_gflops_per_input": 24185.01735334493,
|
| 824 |
+
"latency_ms_per_input": 690.419287848275
|
| 825 |
+
},
|
| 826 |
+
{
|
| 827 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 828 |
+
"pair_label": "Construction / YOLO26x + Qwen3.8",
|
| 829 |
+
"task": "construction",
|
| 830 |
+
"method": "original",
|
| 831 |
+
"method_label": "原方法",
|
| 832 |
+
"nominal_budget": 0.5,
|
| 833 |
+
"seed_count": 3,
|
| 834 |
+
"test_metric": 0.29849806777867033,
|
| 835 |
+
"test_metric_std": 0.0015269090065487179,
|
| 836 |
+
"actual_test_call_fraction": 0.4957833999112295,
|
| 837 |
+
"actual_test_call_fraction_std": 0.00518804692745715,
|
| 838 |
+
"flops_gflops_per_input": 47145.796180556055,
|
| 839 |
+
"latency_ms_per_input": 1320.7697036523023
|
| 840 |
+
},
|
| 841 |
+
{
|
| 842 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 843 |
+
"pair_label": "Construction / YOLO26x + Qwen3.8",
|
| 844 |
+
"task": "construction",
|
| 845 |
+
"method": "original",
|
| 846 |
+
"method_label": "原方法",
|
| 847 |
+
"nominal_budget": 0.75,
|
| 848 |
+
"seed_count": 3,
|
| 849 |
+
"test_metric": 0.28532383356176916,
|
| 850 |
+
"test_metric_std": 0.0029913445567370696,
|
| 851 |
+
"actual_test_call_fraction": 0.7654238792720817,
|
| 852 |
+
"actual_test_call_fraction_std": 0.006553405708048556,
|
| 853 |
+
"flops_gflops_per_input": 72681.1245787817,
|
| 854 |
+
"latency_ms_per_input": 2021.8001431963692
|
| 855 |
+
},
|
| 856 |
+
{
|
| 857 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 858 |
+
"pair_label": "Construction / YOLO26x + Qwen3.8",
|
| 859 |
+
"task": "construction",
|
| 860 |
+
"method": "original",
|
| 861 |
+
"method_label": "原方法",
|
| 862 |
+
"nominal_budget": 0.99,
|
| 863 |
+
"seed_count": 3,
|
| 864 |
+
"test_metric": 0.27053969734098926,
|
| 865 |
+
"test_metric_std": 0.0014164533171887024,
|
| 866 |
+
"actual_test_call_fraction": 0.9897913892587661,
|
| 867 |
+
"actual_test_call_fraction_std": 0.0027895705925380728,
|
| 868 |
+
"flops_gflops_per_input": 93929.03981384847,
|
| 869 |
+
"latency_ms_per_input": 2605.126706471309
|
| 870 |
+
},
|
| 871 |
+
{
|
| 872 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 873 |
+
"pair_label": "Construction / YOLO26x + Qwen3.8",
|
| 874 |
+
"task": "construction",
|
| 875 |
+
"method": "original",
|
| 876 |
+
"method_label": "原方法",
|
| 877 |
+
"nominal_budget": 1.0,
|
| 878 |
+
"seed_count": 3,
|
| 879 |
+
"test_metric": 0.27001009067886245,
|
| 880 |
+
"test_metric_std": 0.0,
|
| 881 |
+
"actual_test_call_fraction": 1.0,
|
| 882 |
+
"actual_test_call_fraction_std": 0.0,
|
| 883 |
+
"flops_gflops_per_input": 94895.80944867841,
|
| 884 |
+
"latency_ms_per_input": 2631.667776610426
|
| 885 |
+
},
|
| 886 |
+
{
|
| 887 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 888 |
+
"pair_label": "Construction / YOLO26x + Qwen3.8",
|
| 889 |
+
"task": "construction",
|
| 890 |
+
"method": "confidence",
|
| 891 |
+
"method_label": "置信度路由",
|
| 892 |
+
"nominal_budget": 0.0,
|
| 893 |
+
"seed_count": 1,
|
| 894 |
+
"test_metric": 0.18129624718318377,
|
| 895 |
+
"test_metric_std": 0.0,
|
| 896 |
+
"actual_test_call_fraction": 0.0,
|
| 897 |
+
"actual_test_call_fraction_std": 0.0,
|
| 898 |
+
"flops_gflops_per_input": 194.418697728,
|
| 899 |
+
"latency_ms_per_input": 31.79686254823173
|
| 900 |
+
},
|
| 901 |
+
{
|
| 902 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 903 |
+
"pair_label": "Construction / YOLO26x + Qwen3.8",
|
| 904 |
+
"task": "construction",
|
| 905 |
+
"method": "confidence",
|
| 906 |
+
"method_label": "置信度路由",
|
| 907 |
+
"nominal_budget": 0.01,
|
| 908 |
+
"seed_count": 1,
|
| 909 |
+
"test_metric": 0.18129624718318377,
|
| 910 |
+
"test_metric_std": 0.0,
|
| 911 |
+
"actual_test_call_fraction": 0.0,
|
| 912 |
+
"actual_test_call_fraction_std": 0.0,
|
| 913 |
+
"flops_gflops_per_input": 194.418697728,
|
| 914 |
+
"latency_ms_per_input": 31.79686254823173
|
| 915 |
+
},
|
| 916 |
+
{
|
| 917 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 918 |
+
"pair_label": "Construction / YOLO26x + Qwen3.8",
|
| 919 |
+
"task": "construction",
|
| 920 |
+
"method": "confidence",
|
| 921 |
+
"method_label": "置信度路由",
|
| 922 |
+
"nominal_budget": 0.05,
|
| 923 |
+
"seed_count": 1,
|
| 924 |
+
"test_metric": 0.18129624718318377,
|
| 925 |
+
"test_metric_std": 0.0,
|
| 926 |
+
"actual_test_call_fraction": 0.0,
|
| 927 |
+
"actual_test_call_fraction_std": 0.0,
|
| 928 |
+
"flops_gflops_per_input": 194.418697728,
|
| 929 |
+
"latency_ms_per_input": 31.79686254823173
|
| 930 |
+
},
|
| 931 |
+
{
|
| 932 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 933 |
+
"pair_label": "Construction / YOLO26x + Qwen3.8",
|
| 934 |
+
"task": "construction",
|
| 935 |
+
"method": "confidence",
|
| 936 |
+
"method_label": "置信度路由",
|
| 937 |
+
"nominal_budget": 0.1,
|
| 938 |
+
"seed_count": 1,
|
| 939 |
+
"test_metric": 0.18129624718318377,
|
| 940 |
+
"test_metric_std": 0.0,
|
| 941 |
+
"actual_test_call_fraction": 0.0,
|
| 942 |
+
"actual_test_call_fraction_std": 0.0,
|
| 943 |
+
"flops_gflops_per_input": 194.418697728,
|
| 944 |
+
"latency_ms_per_input": 31.79686254823173
|
| 945 |
+
},
|
| 946 |
+
{
|
| 947 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 948 |
+
"pair_label": "Construction / YOLO26x + Qwen3.8",
|
| 949 |
+
"task": "construction",
|
| 950 |
+
"method": "confidence",
|
| 951 |
+
"method_label": "置信度路由",
|
| 952 |
+
"nominal_budget": 0.25,
|
| 953 |
+
"seed_count": 1,
|
| 954 |
+
"test_metric": 0.18026802885144627,
|
| 955 |
+
"test_metric_std": 0.0,
|
| 956 |
+
"actual_test_call_fraction": 0.24800266311584554,
|
| 957 |
+
"actual_test_call_fraction_std": 0.0,
|
| 958 |
+
"flops_gflops_per_input": 23680.615804738005,
|
| 959 |
+
"latency_ms_per_input": 676.5717729930834
|
| 960 |
+
},
|
| 961 |
+
{
|
| 962 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 963 |
+
"pair_label": "Construction / YOLO26x + Qwen3.8",
|
| 964 |
+
"task": "construction",
|
| 965 |
+
"method": "confidence",
|
| 966 |
+
"method_label": "置信度路由",
|
| 967 |
+
"nominal_budget": 0.5,
|
| 968 |
+
"seed_count": 1,
|
| 969 |
+
"test_metric": 0.18015785955296285,
|
| 970 |
+
"test_metric_std": 0.0,
|
| 971 |
+
"actual_test_call_fraction": 0.5013315579227696,
|
| 972 |
+
"actual_test_call_fraction_std": 0.0,
|
| 973 |
+
"flops_gflops_per_input": 47671.21446035493,
|
| 974 |
+
"latency_ms_per_input": 1335.1941982931266
|
| 975 |
+
},
|
| 976 |
+
{
|
| 977 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 978 |
+
"pair_label": "Construction / YOLO26x + Qwen3.8",
|
| 979 |
+
"task": "construction",
|
| 980 |
+
"method": "confidence",
|
| 981 |
+
"method_label": "置信度路由",
|
| 982 |
+
"nominal_budget": 0.75,
|
| 983 |
+
"seed_count": 1,
|
| 984 |
+
"test_metric": 0.22103445955201576,
|
| 985 |
+
"test_metric_std": 0.0,
|
| 986 |
+
"actual_test_call_fraction": 0.7486684420772304,
|
| 987 |
+
"actual_test_call_fraction_std": 0.0,
|
| 988 |
+
"flops_gflops_per_input": 71094.36137378908,
|
| 989 |
+
"latency_ms_per_input": 1978.2381693810794
|
| 990 |
+
},
|
| 991 |
+
{
|
| 992 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 993 |
+
"pair_label": "Construction / YOLO26x + Qwen3.8",
|
| 994 |
+
"task": "construction",
|
| 995 |
+
"method": "confidence",
|
| 996 |
+
"method_label": "置信度路由",
|
| 997 |
+
"nominal_budget": 0.99,
|
| 998 |
+
"seed_count": 1,
|
| 999 |
+
"test_metric": 0.2649306125590882,
|
| 1000 |
+
"test_metric_std": 0.0,
|
| 1001 |
+
"actual_test_call_fraction": 0.987017310252996,
|
| 1002 |
+
"actual_test_call_fraction_std": 0.0,
|
| 1003 |
+
"flops_gflops_per_input": 93666.33067394902,
|
| 1004 |
+
"latency_ms_per_input": 2597.9144591508966
|
| 1005 |
+
},
|
| 1006 |
+
{
|
| 1007 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 1008 |
+
"pair_label": "Construction / YOLO26x + Qwen3.8",
|
| 1009 |
+
"task": "construction",
|
| 1010 |
+
"method": "confidence",
|
| 1011 |
+
"method_label": "置信度路由",
|
| 1012 |
+
"nominal_budget": 1.0,
|
| 1013 |
+
"seed_count": 1,
|
| 1014 |
+
"test_metric": 0.27001009067886245,
|
| 1015 |
+
"test_metric_std": 0.0,
|
| 1016 |
+
"actual_test_call_fraction": 1.0,
|
| 1017 |
+
"actual_test_call_fraction_std": 0.0,
|
| 1018 |
+
"flops_gflops_per_input": 94895.80944867841,
|
| 1019 |
+
"latency_ms_per_input": 2631.667776610426
|
| 1020 |
+
}
|
| 1021 |
+
],
|
| 1022 |
+
"comparisons": {
|
| 1023 |
+
"grefcoco|instancevg|MiniCPM": {
|
| 1024 |
+
"both_vs_confidence_metric_delta_percentage_points": 6.175500288309744,
|
| 1025 |
+
"both_vs_confidence_call_delta_percentage_points": -40.80197374018093,
|
| 1026 |
+
"both_vs_confidence_flops_relative_percent": -78.21165529717908,
|
| 1027 |
+
"both_vs_confidence_latency_relative_percent": -76.66417123634449
|
| 1028 |
+
},
|
| 1029 |
+
"construction|yolo26x|Qwen3.8": {
|
| 1030 |
+
"both_vs_confidence_metric_delta_percentage_points": 4.1333041716827665,
|
| 1031 |
+
"both_vs_confidence_call_delta_percentage_points": -48.069241011984026,
|
| 1032 |
+
"both_vs_confidence_flops_relative_percent": -52.01415993620476,
|
| 1033 |
+
"both_vs_confidence_latency_relative_percent": -51.44759733081112
|
| 1034 |
+
}
|
| 1035 |
+
}
|
| 1036 |
+
}
|
releases/20260913_lr/paper_data/cost_20260911/fixed_budget_cost_performance.csv
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pair_key,pair_label,task,method,method_label,nominal_budget,seed_count,test_metric,test_metric_std,actual_test_call_fraction,actual_test_call_fraction_std,flops_gflops_per_input,latency_ms_per_input
|
| 2 |
+
grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,both,两项同时改,0.0,3,0.6695119530385957,0.0,0.0,0.0,7.503634432,16.318855201956467
|
| 3 |
+
grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,both,两项同时改,0.01,3,0.6743328701471798,0.0002504540415259137,0.021268751949635595,0.0024468276041848553,222.01478381055546,47.10392897831558
|
| 4 |
+
grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,both,两项同时改,0.05,3,0.6805811568310505,0.0004271568320976906,0.11976670542305912,0.0032185843726325893,1215.4397511547988,189.67302620038757
|
| 5 |
+
grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,both,两项同时改,0.1,3,0.6804866290446077,0.0005385595679416328,0.172031118547297,0.0033259813564282133,1742.5651488943693,265.32221416016074
|
| 6 |
+
grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,both,两项同时改,0.25,3,0.6794184650578038,0.00038210745789084077,0.24530905859777485,0.009454413833556307,2481.627562219953,371.38705501097667
|
| 7 |
+
grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,both,两项同时改,0.5,3,0.664568149807636,0.0016062493473305785,0.4310372534006371,0.0088737075287505,4354.834256882111,640.2160014811561
|
| 8 |
+
grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,both,两项同时改,0.75,3,0.6150355897115957,0.0013848240058531283,0.7057350008034863,0.006014701362190238,7125.364923966921,1037.8223321215987
|
| 9 |
+
grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,both,两项同时改,0.99,3,0.5069147075782926,9.357773432602366e-05,0.988051687793627,0.00048144255369636903,9972.73825167356,1446.4565603014837
|
| 10 |
+
grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,both,两项同时改,1.0,3,0.5000708958398321,0.0,1.0,0.0,10093.24584848,1463.7509306362917
|
| 11 |
+
grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,original,原方法,0.0,3,0.6695119530385957,0.0,0.0,0.0,7.503634432,16.318855201956467
|
| 12 |
+
grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,original,原方法,0.01,3,0.6742194368034484,0.0005415378002401663,0.04492905689627466,0.0027758676387843045,460.64652020812184,81.35061327263864
|
| 13 |
+
grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,original,原方法,0.05,3,0.679891103990018,0.0007177890142290943,0.10489748461560276,0.007840998690258661,1065.4726231670354,168.15083906695963
|
| 14 |
+
grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,original,原方法,0.1,3,0.680127423456125,0.0007907632127485806,0.16706840975904869,0.004046760924054268,1692.5125473727062,258.1390302790103
|
| 15 |
+
grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,original,原方法,0.25,3,0.6658726332605469,0.0048175982262131,0.3117620924670807,0.009206880232216604,3151.855731167172,467.5733077433342
|
| 16 |
+
grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,original,原方法,0.5,3,0.6273242019491629,0.009832598413097358,0.5310665570144345,0.009248673211785196,5363.704026981612,785.0016240151261
|
| 17 |
+
grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,original,原方法,0.75,3,0.5814404144098159,0.00858033252822142,0.7394719677849304,0.008502993761525655,7465.627276025616,1086.65430025841
|
| 18 |
+
grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,original,原方法,0.99,3,0.5100152189736173,0.0036056620177345906,0.978079006323909,0.009338895613008017,9872.15635718717,1432.0217812641245
|
| 19 |
+
grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,original,原方法,1.0,3,0.5000708958398321,0.0,1.0,0.0,10093.24584848,1463.7509306362917
|
| 20 |
+
grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,confidence,置信度路由,0.0,1,0.6695119530385957,0.0,0.0,0.0,7.503634432,16.318855201956467
|
| 21 |
+
grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,confidence,置信度路由,0.01,1,0.6686044862887446,0.0,0.002467175226157729,0.0,32.38692775991243,19.889923760014124
|
| 22 |
+
grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,confidence,置信度路由,0.05,1,0.6662791027422511,0.0,0.012534384482318578,0.0,133.9222051324287,34.46152534749078
|
| 23 |
+
grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,confidence,置信度路由,0.1,1,0.6645208859144145,0.0,0.0345971698380739,0.0,356.4417707544502,66.39590854483396
|
| 24 |
+
grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,confidence,置信度路由,0.25,1,0.6596999688058305,0.0,0.12477667810452883,0.0,1265.9690441195253,196.92462135659662
|
| 25 |
+
grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,confidence,置信度路由,0.5,1,0.6527521765022829,0.0,0.2345234381646485,0.0,2372.8465749128713,355.77560204260965
|
| 26 |
+
grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,confidence,置信度路由,0.75,1,0.567762243711539,0.0,0.7478093185491875,0.0,7549.715646582009,1098.722049178743
|
| 27 |
+
grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,confidence,置信度路由,0.99,1,0.5019425460114001,0.0,0.9908118991577575,0.0,10000.577031948465,1450.4517787649047
|
| 28 |
+
grefcoco|instancevg|MiniCPM,gRefCOCO / InstanceVG + MiniCPM,grefcoco,confidence,置信度路由,1.0,1,0.5000708958398321,0.0,1.0,0.0,10093.24584848,1463.7509306362917
|
| 29 |
+
construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,both,两项同时改,0.0,3,0.18129624718318374,2.7755575615628914e-17,0.0,0.0,194.418697728,31.79686254823173
|
| 30 |
+
construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,both,两项同时改,0.01,3,0.1834834493716783,0.0012311365629078506,0.01187305814469596,0.0025254786233335643,1318.8138164976065,62.665281079596
|
| 31 |
+
construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,both,两项同时改,0.05,3,0.19244312207391248,0.0014149660525805208,0.04804704837993786,0.0025254786233335656,4744.541000786314,156.71298613777122
|
| 32 |
+
construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,both,两项同时改,0.1,3,0.2058996264309502,0.000756529123691676,0.10707944962272525,0.0037002512207130658,10334.991497846413,310.1896091161431
|
| 33 |
+
construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,both,两项同时改,0.25,3,0.25617136079673564,0.0020077364088740926,0.26498002663115844,0.008293336414138841,25288.39574092258,720.7107265940061
|
| 34 |
+
construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,both,两项同时改,0.5,3,0.29980106360416753,0.002221016752728669,0.49511762094984463,0.00440231567351916,47082.74598698018,1319.038764295403
|
| 35 |
+
construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,both,两项同时改,0.75,3,0.28788636673929996,0.0011284209510194398,0.7496671105193076,0.005429263125932131,71188.93666415288,1980.8345784164278
|
| 36 |
+
construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,both,两项同时改,0.99,3,0.27239176103160984,0.0005578690368974412,0.9862405681313805,0.0023116585281845767,93592.7721147772,2595.8950299011817
|
| 37 |
+
construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,both,两项同时改,1.0,3,0.27001009067886245,0.0,1.0,0.0,94895.80944867841,2631.667776610426
|
| 38 |
+
construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,original,原方法,0.0,3,0.18129624718318374,2.7755575615628914e-17,0.0,0.0,194.418697728,31.79686254823173
|
| 39 |
+
construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,original,原方法,0.01,3,0.18300903853620068,0.000992883110506064,0.010319573901464712,0.0032954377285658006,1171.6966981539194,58.626422580165155
|
| 40 |
+
construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,original,原方法,0.05,3,0.19493527207894204,0.0032238119689520237,0.049822458943630715,0.011397380483895993,4912.674850321956,161.32882442283505
|
| 41 |
+
construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,original,原方法,0.1,3,0.20346296196880112,0.004110692938093332,0.09576120727918332,0.006259345218007303,9263.138207056696,280.76364004886125
|
| 42 |
+
construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,original,原方法,0.25,3,0.24461408052105227,0.0022654085644492608,0.2533288948069241,0.0017823322685540256,24185.01735334493,690.419287848275
|
| 43 |
+
construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,original,原方法,0.5,3,0.29849806777867033,0.0015269090065487179,0.4957833999112295,0.00518804692745715,47145.796180556055,1320.7697036523023
|
| 44 |
+
construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,original,原方法,0.75,3,0.28532383356176916,0.0029913445567370696,0.7654238792720817,0.006553405708048556,72681.1245787817,2021.8001431963692
|
| 45 |
+
construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,original,原方法,0.99,3,0.27053969734098926,0.0014164533171887024,0.9897913892587661,0.0027895705925380728,93929.03981384847,2605.126706471309
|
| 46 |
+
construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,original,原方法,1.0,3,0.27001009067886245,0.0,1.0,0.0,94895.80944867841,2631.667776610426
|
| 47 |
+
construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,confidence,置信度路由,0.0,1,0.18129624718318377,0.0,0.0,0.0,194.418697728,31.79686254823173
|
| 48 |
+
construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,confidence,置信度路由,0.01,1,0.18129624718318377,0.0,0.0,0.0,194.418697728,31.79686254823173
|
| 49 |
+
construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,confidence,置信度路由,0.05,1,0.18129624718318377,0.0,0.0,0.0,194.418697728,31.79686254823173
|
| 50 |
+
construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,confidence,置信度路由,0.1,1,0.18129624718318377,0.0,0.0,0.0,194.418697728,31.79686254823173
|
| 51 |
+
construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,confidence,置信度路由,0.25,1,0.18026802885144627,0.0,0.24800266311584554,0.0,23680.615804738005,676.5717729930834
|
| 52 |
+
construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,confidence,置信度路由,0.5,1,0.18015785955296285,0.0,0.5013315579227696,0.0,47671.21446035493,1335.1941982931266
|
| 53 |
+
construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,confidence,置信度路由,0.75,1,0.22103445955201576,0.0,0.7486684420772304,0.0,71094.36137378908,1978.2381693810794
|
| 54 |
+
construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,confidence,置信度路由,0.99,1,0.2649306125590882,0.0,0.987017310252996,0.0,93666.33067394902,2597.9144591508966
|
| 55 |
+
construction|yolo26x|Qwen3.8,Construction / YOLO26x + Qwen3.8,construction,confidence,置信度路由,1.0,1,0.27001009067886245,0.0,1.0,0.0,94895.80944867841,2631.667776610426
|
releases/20260913_lr/paper_data/cost_20260911/full_cost_performance_curves.csv
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
releases/20260913_lr/paper_data/cost_20260911/grefcoco_router.json
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"component": "router",
|
| 3 |
+
"expert": "instancevg",
|
| 4 |
+
"task": "grefcoco",
|
| 5 |
+
"endpoint": "MiniCPM",
|
| 6 |
+
"condition": "both",
|
| 7 |
+
"seed": 42,
|
| 8 |
+
"device": "cuda:0",
|
| 9 |
+
"checkpoint": "/mnt/HithinkOmniSSD/user_workspace/caisihang/research/ICASSP_ModelCollaboration/outputs/router_method_improvement_20260911_r1/conditions/both_instancevg_MiniCPM_seed42/best.pt",
|
| 10 |
+
"sample_count": 1024,
|
| 11 |
+
"batch_size": 256,
|
| 12 |
+
"row_count_loaded": 35263,
|
| 13 |
+
"mean_latency_ms_per_sample": 0.001830718741985038,
|
| 14 |
+
"median_latency_ms_per_sample": 0.0018171250121667981,
|
| 15 |
+
"latency_ms_per_sample": [
|
| 16 |
+
0.001949499943293631,
|
| 17 |
+
0.0018388750031590462,
|
| 18 |
+
0.00179537502117455,
|
| 19 |
+
0.0017391250003129244
|
| 20 |
+
],
|
| 21 |
+
"flops_counted_per_batch": 0,
|
| 22 |
+
"gflops_counted_per_sample": 0.0,
|
| 23 |
+
"operators": {},
|
| 24 |
+
"flops_scope": "GPU FlopCounterMode for the router forward; multiply-add counted as 2 FLOPs.",
|
| 25 |
+
"latency_scope": "A800 CUDA forward latency on cached router features, excluding feature loading and model load.",
|
| 26 |
+
"completed": true
|
| 27 |
+
}
|
releases/20260913_lr/paper_data/cost_20260911/instancevg_specialist.json
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"component": "specialist",
|
| 3 |
+
"expert": "instancevg",
|
| 4 |
+
"task": "grefcoco",
|
| 5 |
+
"device": "cuda:0",
|
| 6 |
+
"checkpoint": "/mnt/HithinkOmniSSD/user_workspace/caisihang/research/ICASSP_ModelCollaboration/models/grefcoco/instancevg/InstanceVG-grefcoco.pth",
|
| 7 |
+
"sample_count": 64,
|
| 8 |
+
"batch_size": 16,
|
| 9 |
+
"sample_ids": [
|
| 10 |
+
"testA-30844-84861",
|
| 11 |
+
"testB-44353-122077",
|
| 12 |
+
"testB-36889-101643",
|
| 13 |
+
"testA-49055-135087",
|
| 14 |
+
"testA-81245-223633",
|
| 15 |
+
"testB-37669-103833",
|
| 16 |
+
"testA-5038-13943",
|
| 17 |
+
"testB-79646-219274",
|
| 18 |
+
"testA-469-1309",
|
| 19 |
+
"testB-46644-128438",
|
| 20 |
+
"testA-24562-67515",
|
| 21 |
+
"testA-47070-129599",
|
| 22 |
+
"testA-54554-150141",
|
| 23 |
+
"testB-12382-34099",
|
| 24 |
+
"testB-23014-63279",
|
| 25 |
+
"testB-79647-219275",
|
| 26 |
+
"testB-69479-191564",
|
| 27 |
+
"testA-15582-42797",
|
| 28 |
+
"testB-56692-156000",
|
| 29 |
+
"testB-42726-117660",
|
| 30 |
+
"testA-80455-221494",
|
| 31 |
+
"testB-33115-91174",
|
| 32 |
+
"testA-62940-173455",
|
| 33 |
+
"testB-31494-86670",
|
| 34 |
+
"testA-54586-150233",
|
| 35 |
+
"testA-81445-224177",
|
| 36 |
+
"testB-8417-23141",
|
| 37 |
+
"testA-24438-67190",
|
| 38 |
+
"testA-59285-163255",
|
| 39 |
+
"testA-73258-201893",
|
| 40 |
+
"testB-945-2633",
|
| 41 |
+
"testA-54307-149426",
|
| 42 |
+
"testB-60557-166822",
|
| 43 |
+
"testA-81622-224665",
|
| 44 |
+
"testA-69466-191525",
|
| 45 |
+
"testA-60731-167321",
|
| 46 |
+
"testA-7857-21644",
|
| 47 |
+
"testA-44142-121499",
|
| 48 |
+
"testB-66162-182407",
|
| 49 |
+
"testA-31764-87406",
|
| 50 |
+
"testB-38637-106482",
|
| 51 |
+
"testB-52921-145709",
|
| 52 |
+
"testB-65796-181394",
|
| 53 |
+
"testB-2936-8113",
|
| 54 |
+
"testA-79438-218690",
|
| 55 |
+
"testA-57397-158016",
|
| 56 |
+
"testB-23920-65743",
|
| 57 |
+
"testB-15419-42358",
|
| 58 |
+
"testA-10078-27739",
|
| 59 |
+
"testA-43890-120793",
|
| 60 |
+
"testA-55116-151689",
|
| 61 |
+
"testB-27487-75582",
|
| 62 |
+
"testA-26856-73830",
|
| 63 |
+
"testB-50845-140107",
|
| 64 |
+
"testA-29199-80374",
|
| 65 |
+
"testB-65935-181778",
|
| 66 |
+
"testA-17736-48684",
|
| 67 |
+
"testB-69477-191553",
|
| 68 |
+
"testB-22290-61227",
|
| 69 |
+
"testA-58476-160950",
|
| 70 |
+
"testB-36265-99859",
|
| 71 |
+
"testA-5734-15837",
|
| 72 |
+
"testB-67952-187353",
|
| 73 |
+
"testB-47054-129555"
|
| 74 |
+
],
|
| 75 |
+
"model_load_seconds_excluded_from_latency": true,
|
| 76 |
+
"forward_latency_ms_per_sample": [
|
| 77 |
+
11.21629810333252,
|
| 78 |
+
11.368107795715332,
|
| 79 |
+
11.16999626159668,
|
| 80 |
+
11.233466148376465
|
| 81 |
+
],
|
| 82 |
+
"end_to_end_latency_ms_per_sample": [
|
| 83 |
+
16.296884561597835,
|
| 84 |
+
16.604527561867144,
|
| 85 |
+
15.993722627172247,
|
| 86 |
+
16.3729631822207
|
| 87 |
+
],
|
| 88 |
+
"mean_forward_latency_ms_per_sample": 11.246967077255249,
|
| 89 |
+
"median_forward_latency_ms_per_sample": 11.224882125854492,
|
| 90 |
+
"mean_end_to_end_latency_ms_per_sample": 16.31702448321448,
|
| 91 |
+
"median_end_to_end_latency_ms_per_sample": 16.334923871909268,
|
| 92 |
+
"flop_profiles": [
|
| 93 |
+
{
|
| 94 |
+
"batch_size": 16,
|
| 95 |
+
"flops_counted": 120042881024,
|
| 96 |
+
"gflops_counted_per_sample": 7.502680064,
|
| 97 |
+
"operators": {
|
| 98 |
+
"aten.bmm": 120010113024,
|
| 99 |
+
"aten.baddbmm": 32768000
|
| 100 |
+
}
|
| 101 |
+
},
|
| 102 |
+
{
|
| 103 |
+
"batch_size": 16,
|
| 104 |
+
"flops_counted": 120042881024,
|
| 105 |
+
"gflops_counted_per_sample": 7.502680064,
|
| 106 |
+
"operators": {
|
| 107 |
+
"aten.bmm": 120010113024,
|
| 108 |
+
"aten.baddbmm": 32768000
|
| 109 |
+
}
|
| 110 |
+
},
|
| 111 |
+
{
|
| 112 |
+
"batch_size": 16,
|
| 113 |
+
"flops_counted": 120042881024,
|
| 114 |
+
"gflops_counted_per_sample": 7.502680064,
|
| 115 |
+
"operators": {
|
| 116 |
+
"aten.bmm": 120010113024,
|
| 117 |
+
"aten.baddbmm": 32768000
|
| 118 |
+
}
|
| 119 |
+
},
|
| 120 |
+
{
|
| 121 |
+
"batch_size": 16,
|
| 122 |
+
"flops_counted": 120042881024,
|
| 123 |
+
"gflops_counted_per_sample": 7.502680064,
|
| 124 |
+
"operators": {
|
| 125 |
+
"aten.bmm": 120010113024,
|
| 126 |
+
"aten.baddbmm": 32768000
|
| 127 |
+
}
|
| 128 |
+
}
|
| 129 |
+
],
|
| 130 |
+
"mean_counted_gflops_per_sample": 7.502680064,
|
| 131 |
+
"flops_scope": "GPU FlopCounterMode dense products, convolutions and supported attention; multiply-add counted as 2 FLOPs. Elementwise, normalization, custom ops and postprocessing may be missing.",
|
| 132 |
+
"latency_scope": "A800 wall-clock CUDA forward latency and end-to-end latency including image read/preprocess, excluding model load; batch processing uses the recorded batch size.",
|
| 133 |
+
"completed": true
|
| 134 |
+
}
|
releases/20260913_lr/paper_data/cost_20260911/make_cost_analysis_20260911.py
ADDED
|
@@ -0,0 +1,457 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Build the A800 cost--performance supplement for two representative pairs.
|
| 2 |
+
|
| 3 |
+
The script reads the frozen test curves and the measured A800 component records.
|
| 4 |
+
It does not select a test-set operating point: the selected points are copied from
|
| 5 |
+
the validation-locked records, and all curve values retain their nominal budget.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import argparse
|
| 11 |
+
import csv
|
| 12 |
+
import json
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
|
| 15 |
+
import matplotlib.pyplot as plt
|
| 16 |
+
import numpy as np
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
PAIR_FILES = {
|
| 20 |
+
"grefcoco|instancevg|MiniCPM": {
|
| 21 |
+
"label": "gRefCOCO / InstanceVG + MiniCPM",
|
| 22 |
+
"specialist": "instancevg_specialist.json",
|
| 23 |
+
"router": "grefcoco_router.json",
|
| 24 |
+
"large_latency": "minicpm_grefcoco.json",
|
| 25 |
+
"large_flops": "minicpm_grefcoco_flops.json",
|
| 26 |
+
},
|
| 27 |
+
"construction|yolo26x|Qwen3.8": {
|
| 28 |
+
"label": "Construction / YOLO26x + Qwen3.8",
|
| 29 |
+
"specialist": "yolo26x_specialist.json",
|
| 30 |
+
"router": "construction_router.json",
|
| 31 |
+
"large_latency": "qwen_construction.json",
|
| 32 |
+
"large_flops": "qwen_construction_flops.json",
|
| 33 |
+
},
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
METHOD_LABELS = {
|
| 37 |
+
"both": "两项同时改",
|
| 38 |
+
"original": "原方法",
|
| 39 |
+
"confidence": "置信度路由",
|
| 40 |
+
}
|
| 41 |
+
PLOT_LABELS = {"both": "Two changes", "original": "Original", "confidence": "Confidence"}
|
| 42 |
+
PLOT_METHODS = ("both", "original", "confidence")
|
| 43 |
+
PLOT_COLORS = {"both": "#2166ac", "original": "#d6604d", "confidence": "#252525"}
|
| 44 |
+
PLOT_STYLES = {"both": "-", "original": "--", "confidence": ":"}
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def read_json(path: Path):
|
| 48 |
+
return json.loads(path.read_text(encoding="utf-8"))
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def write_json(path: Path, value) -> None:
|
| 52 |
+
path.write_text(json.dumps(value, ensure_ascii=False, indent=2), encoding="utf-8")
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def mean_std(values):
|
| 56 |
+
values = np.asarray(values, dtype=float)
|
| 57 |
+
return float(values.mean()), float(values.std(ddof=0))
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def router_flops(architecture: dict) -> tuple[int, dict[str, int]]:
|
| 61 |
+
d = int(architecture["representation_dim"])
|
| 62 |
+
branch_dims = architecture["input_dimensions"]
|
| 63 |
+
branch_costs = {}
|
| 64 |
+
for branch in architecture["branches"]:
|
| 65 |
+
input_key = "image_flat_plus_valid" if branch == "image" else branch
|
| 66 |
+
branch_costs[branch] = 2 * int(branch_dims[input_key]) * d
|
| 67 |
+
head_input = len(architecture["branches"]) * d
|
| 68 |
+
branch_costs["head_input_to_hidden"] = 2 * head_input * int(architecture["head_hidden_dim"])
|
| 69 |
+
branch_costs["head_hidden_to_classes"] = 2 * int(architecture["head_hidden_dim"]) * int(architecture["class_count"])
|
| 70 |
+
return sum(branch_costs.values()), branch_costs
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def build_cost_components(root: Path, architectures: list[dict]) -> dict[str, dict]:
|
| 74 |
+
components = {}
|
| 75 |
+
for architecture in architectures:
|
| 76 |
+
key = architecture["pair_key"]
|
| 77 |
+
files = PAIR_FILES[key]
|
| 78 |
+
specialist = read_json(root / files["specialist"])
|
| 79 |
+
router = read_json(root / files["router"])
|
| 80 |
+
large_latency = read_json(root / files["large_latency"])
|
| 81 |
+
large_flops = read_json(root / files["large_flops"])
|
| 82 |
+
router_flop_count, router_breakdown = router_flops(architecture)
|
| 83 |
+
completion_tokens = [
|
| 84 |
+
output["completion_tokens"]
|
| 85 |
+
for batch in large_latency["batches"]
|
| 86 |
+
for output in batch["outputs"]
|
| 87 |
+
]
|
| 88 |
+
prefill = float(large_flops["mean_prefill_gflops_counted"])
|
| 89 |
+
decode = float(large_flops["mean_decode_one_token_gflops_counted"])
|
| 90 |
+
mean_tokens = float(large_latency["mean_completion_tokens"])
|
| 91 |
+
large_estimated = prefill + decode * mean_tokens
|
| 92 |
+
components[key] = {
|
| 93 |
+
"pair_key": key,
|
| 94 |
+
"task": architecture["task"],
|
| 95 |
+
"expert": architecture["expert"],
|
| 96 |
+
"endpoint": architecture["endpoint"],
|
| 97 |
+
"label": PAIR_FILES[key]["label"],
|
| 98 |
+
"specialist_gflops_per_input": float(specialist["mean_counted_gflops_per_sample"]),
|
| 99 |
+
"specialist_forward_latency_ms": float(specialist["mean_forward_latency_ms_per_sample"]),
|
| 100 |
+
"specialist_end_to_end_latency_ms": float(specialist["mean_end_to_end_latency_ms_per_sample"]),
|
| 101 |
+
"specialist_sample_count": int(specialist["sample_count"]),
|
| 102 |
+
"router_gflops_per_input": router_flop_count / 1e9,
|
| 103 |
+
"router_flop_count": router_flop_count,
|
| 104 |
+
"router_flop_breakdown": router_breakdown,
|
| 105 |
+
"router_measured_gflops_per_input": float(router["gflops_counted_per_sample"]),
|
| 106 |
+
"router_latency_ms": float(router["mean_latency_ms_per_sample"]),
|
| 107 |
+
"router_sample_count": int(router["sample_count"]),
|
| 108 |
+
"large_prefill_gflops": prefill,
|
| 109 |
+
"large_decode_one_token_gflops": decode,
|
| 110 |
+
"large_mean_completion_tokens": mean_tokens,
|
| 111 |
+
"large_completion_token_min": int(min(completion_tokens)),
|
| 112 |
+
"large_completion_token_max": int(max(completion_tokens)),
|
| 113 |
+
"large_estimated_full_generation_gflops": large_estimated,
|
| 114 |
+
"large_batch_wall_latency_ms_per_input": float(large_latency["mean_batch_wall_ms_per_sample"]),
|
| 115 |
+
"large_measured_sample_count": int(large_latency["measured_count"]),
|
| 116 |
+
"large_flops_probe_count": int(large_flops["sample_count"]),
|
| 117 |
+
"flops_formula": "specialist + router + actual_call_fraction * (prefill + mean_completion_tokens * one_token_decode)",
|
| 118 |
+
"latency_formula": "specialist end-to-end + router + actual_call_fraction * large-model batch-wall latency",
|
| 119 |
+
"specialist_flops_scope": specialist["flops_scope"],
|
| 120 |
+
"specialist_latency_scope": specialist["latency_scope"],
|
| 121 |
+
"large_flops_scope": large_flops["scope"],
|
| 122 |
+
"large_latency_scope": large_latency["latency_scope"],
|
| 123 |
+
}
|
| 124 |
+
return components
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def load_relevant_groups(results: dict, pair_key: str) -> dict[str, list[dict]]:
|
| 128 |
+
task, expert, endpoint = pair_key.split("|")
|
| 129 |
+
groups = {
|
| 130 |
+
method: []
|
| 131 |
+
for method in PLOT_METHODS
|
| 132 |
+
}
|
| 133 |
+
for group in results["groups"]:
|
| 134 |
+
if (group["task"], group["expert"], group["endpoint"]) != (task, expert, endpoint):
|
| 135 |
+
continue
|
| 136 |
+
if group["method"] in groups:
|
| 137 |
+
groups[group["method"]].append(group)
|
| 138 |
+
for method, entries in groups.items():
|
| 139 |
+
if not entries:
|
| 140 |
+
raise ValueError(f"missing {method} group for {pair_key}")
|
| 141 |
+
return groups
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
def curve_rows(groups: dict[str, list[dict]], pair_key: str, components: dict) -> list[dict]:
|
| 145 |
+
rows = []
|
| 146 |
+
for method, entries in groups.items():
|
| 147 |
+
per_budget = {}
|
| 148 |
+
for group in entries:
|
| 149 |
+
curve = group["deployments"]["endpoint"]["test_curve"]
|
| 150 |
+
for point in curve:
|
| 151 |
+
budget = round(float(point["budget"]), 10)
|
| 152 |
+
per_budget.setdefault(budget, []).append(point)
|
| 153 |
+
for budget in sorted(per_budget):
|
| 154 |
+
points = per_budget[budget]
|
| 155 |
+
metrics = [float(point["metric"]) for point in points]
|
| 156 |
+
calls = [float(point["actual_fraction"]) for point in points]
|
| 157 |
+
metric_mean, metric_std = mean_std(metrics)
|
| 158 |
+
call_mean, call_std = mean_std(calls)
|
| 159 |
+
component = components[pair_key]
|
| 160 |
+
rows.append({
|
| 161 |
+
"pair_key": pair_key,
|
| 162 |
+
"pair_label": component["label"],
|
| 163 |
+
"task": component["task"],
|
| 164 |
+
"method": method,
|
| 165 |
+
"method_label": METHOD_LABELS[method],
|
| 166 |
+
"nominal_budget": budget,
|
| 167 |
+
"seed_count": len(points),
|
| 168 |
+
"test_metric": metric_mean,
|
| 169 |
+
"test_metric_std": metric_std,
|
| 170 |
+
"actual_test_call_fraction": call_mean,
|
| 171 |
+
"actual_test_call_fraction_std": call_std,
|
| 172 |
+
"flops_gflops_per_input": component["specialist_gflops_per_input"] + component["router_gflops_per_input"] + call_mean * component["large_estimated_full_generation_gflops"],
|
| 173 |
+
"latency_ms_per_input": component["specialist_end_to_end_latency_ms"] + component["router_latency_ms"] + call_mean * component["large_batch_wall_latency_ms_per_input"],
|
| 174 |
+
})
|
| 175 |
+
return rows
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def selected_rows(groups: dict[str, list[dict]], pair_key: str, components: dict) -> list[dict]:
|
| 179 |
+
rows = []
|
| 180 |
+
component = components[pair_key]
|
| 181 |
+
for method, entries in groups.items():
|
| 182 |
+
points = [entry["deployments"]["endpoint"]["test_point"] for entry in entries]
|
| 183 |
+
metrics = [float(point["metric"]) for point in points]
|
| 184 |
+
calls = [float(point["actual_fraction"]) for point in points]
|
| 185 |
+
budgets = [float(point["budget"]) for point in points]
|
| 186 |
+
metric_mean, metric_std = mean_std(metrics)
|
| 187 |
+
call_mean, call_std = mean_std(calls)
|
| 188 |
+
rows.append({
|
| 189 |
+
"pair_key": pair_key,
|
| 190 |
+
"pair_label": component["label"],
|
| 191 |
+
"task": component["task"],
|
| 192 |
+
"method": method,
|
| 193 |
+
"method_label": METHOD_LABELS[method],
|
| 194 |
+
"seed_count": len(points),
|
| 195 |
+
"selected_nominal_budget_mean": float(np.mean(budgets)),
|
| 196 |
+
"selected_nominal_budgets": budgets,
|
| 197 |
+
"test_metric": metric_mean,
|
| 198 |
+
"test_metric_std": metric_std,
|
| 199 |
+
"actual_test_call_fraction": call_mean,
|
| 200 |
+
"actual_test_call_fraction_std": call_std,
|
| 201 |
+
"flops_gflops_per_input": component["specialist_gflops_per_input"] + component["router_gflops_per_input"] + call_mean * component["large_estimated_full_generation_gflops"],
|
| 202 |
+
"latency_ms_per_input": component["specialist_end_to_end_latency_ms"] + component["router_latency_ms"] + call_mean * component["large_batch_wall_latency_ms_per_input"],
|
| 203 |
+
})
|
| 204 |
+
return rows
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def fixed_budget_rows(curves: list[dict], budgets=(0.0, 0.01, 0.05, 0.10, 0.25, 0.50, 0.75, 0.99, 1.0)) -> list[dict]:
|
| 208 |
+
wanted = {round(float(budget), 10) for budget in budgets}
|
| 209 |
+
return [row for row in curves if round(float(row["nominal_budget"]), 10) in wanted]
|
| 210 |
+
|
| 211 |
+
|
| 212 |
+
def write_csv(path: Path, rows: list[dict]) -> None:
|
| 213 |
+
if not rows:
|
| 214 |
+
raise ValueError(f"no rows for {path}")
|
| 215 |
+
fields = list(rows[0])
|
| 216 |
+
with path.open("w", encoding="utf-8", newline="") as handle:
|
| 217 |
+
writer = csv.DictWriter(handle, fieldnames=fields)
|
| 218 |
+
writer.writeheader()
|
| 219 |
+
writer.writerows(rows)
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
def make_plot(curves: list[dict], selected: list[dict], out_path: Path, cost_field: str, x_label: str, log_x: bool) -> None:
|
| 223 |
+
fig, axes = plt.subplots(1, 2, figsize=(7.2, 2.9), constrained_layout=True)
|
| 224 |
+
pair_keys = list(PAIR_FILES)
|
| 225 |
+
for index, (ax, pair_key) in enumerate(zip(axes, pair_keys)):
|
| 226 |
+
pair_rows = [row for row in curves if row["pair_key"] == pair_key]
|
| 227 |
+
all_y = []
|
| 228 |
+
for method in PLOT_METHODS:
|
| 229 |
+
method_rows = [row for row in pair_rows if row["method"] == method]
|
| 230 |
+
x = np.asarray([row[cost_field] for row in method_rows], dtype=float)
|
| 231 |
+
y = 100 * np.asarray([row["test_metric"] for row in method_rows], dtype=float)
|
| 232 |
+
y_std = 100 * np.asarray([row["test_metric_std"] for row in method_rows], dtype=float)
|
| 233 |
+
all_y.extend((y - y_std).tolist())
|
| 234 |
+
all_y.extend((y + y_std).tolist())
|
| 235 |
+
label = PLOT_LABELS[method]
|
| 236 |
+
ax.plot(x, y, PLOT_STYLES[method], color=PLOT_COLORS[method], linewidth=1.25, label=label)
|
| 237 |
+
if method in ("both", "original") and np.any(y_std > 0):
|
| 238 |
+
ax.fill_between(x, y - y_std, y + y_std, color=PLOT_COLORS[method], alpha=0.10, linewidth=0)
|
| 239 |
+
selected_row = next(row for row in selected if row["pair_key"] == pair_key and row["method"] == method)
|
| 240 |
+
ax.scatter(
|
| 241 |
+
[selected_row[cost_field]],
|
| 242 |
+
[100 * selected_row["test_metric"]],
|
| 243 |
+
color=PLOT_COLORS[method],
|
| 244 |
+
marker="*",
|
| 245 |
+
s=42,
|
| 246 |
+
edgecolor="white",
|
| 247 |
+
linewidth=0.45,
|
| 248 |
+
zorder=5,
|
| 249 |
+
)
|
| 250 |
+
y_min, y_max = min(all_y), max(all_y)
|
| 251 |
+
y_pad = max(0.05 * (y_max - y_min), 0.2)
|
| 252 |
+
ax.set_ylim(y_min - y_pad, y_max + 2.0 * y_pad)
|
| 253 |
+
task_label = "gRefCOCO" if PAIR_FILES[pair_key]["label"].startswith("gRefCOCO") else "Construction"
|
| 254 |
+
ax.text(0.02, 0.98, f"({'ab'[index]}) {task_label}", transform=ax.transAxes, ha="left", va="top", fontsize=8.5)
|
| 255 |
+
ax.set_xlabel(x_label)
|
| 256 |
+
ax.set_ylabel("Test performance (%)")
|
| 257 |
+
ax.grid(True, color="#d9d9d9", linewidth=0.45, alpha=0.65)
|
| 258 |
+
ax.spines["top"].set_visible(False)
|
| 259 |
+
ax.spines["right"].set_visible(False)
|
| 260 |
+
if log_x:
|
| 261 |
+
ax.set_xscale("log")
|
| 262 |
+
ax.tick_params(labelsize=8)
|
| 263 |
+
handles, labels = axes[0].get_legend_handles_labels()
|
| 264 |
+
fig.legend(handles, labels, loc="lower center", bbox_to_anchor=(0.5, -0.045), ncol=3, frameon=False, fontsize=8)
|
| 265 |
+
fig.savefig(out_path, format="pdf", bbox_inches="tight", pad_inches=0.04)
|
| 266 |
+
fig.savefig(out_path.with_suffix(".png"), format="png", dpi=300, bbox_inches="tight", pad_inches=0.04)
|
| 267 |
+
plt.close(fig)
|
| 268 |
+
|
| 269 |
+
|
| 270 |
+
def fmt(value: float, digits=3) -> str:
|
| 271 |
+
return f"{value:.{digits}f}"
|
| 272 |
+
|
| 273 |
+
|
| 274 |
+
def pct(value: float, digits=2) -> str:
|
| 275 |
+
return f"{100 * value:.{digits}f}%"
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
def report_markdown(components: dict, selected: list[dict], fixed: list[dict], root: Path, summary: dict) -> str:
|
| 279 |
+
selected_by_key = {(row["pair_key"], row["method"]): row for row in selected}
|
| 280 |
+
lines = [
|
| 281 |
+
"# A800成本—性能补充报告(2026-09-11)",
|
| 282 |
+
"",
|
| 283 |
+
"## 1. 做了什么",
|
| 284 |
+
"",
|
| 285 |
+
"- 在A800上补测了正文中两个代表性模型对:gRefCOCO的InstanceVG + MiniCPM,以及Construction的YOLO26x + Qwen3.8。",
|
| 286 |
+
"- 分别实测了小模型端、路由器端和大模型端的延迟与可计数运算量;大模型端延迟采用批处理均摊后的真实墙钟时间。",
|
| 287 |
+
"- 从已经锁定的测试曲线中提取代表性调用预算,按测试集实际调用率换算每个输入的成本,并生成完整曲线。没有用测试集重新挑选模型、阈值或预算。",
|
| 288 |
+
"",
|
| 289 |
+
"## 2. 为什么做",
|
| 290 |
+
"",
|
| 291 |
+
"正文原有成本图只是代理估算,缺少InstanceVG、完整解码和真实延迟。此次补测用于回答一个更直接的问题:两项同时改相对置信度路由的性能提升,是否值得它增加的计算和时间成本。",
|
| 292 |
+
"",
|
| 293 |
+
"## 3. 原始实测成本",
|
| 294 |
+
"",
|
| 295 |
+
"| 模型对 | 小模型端到端延迟(毫秒/样本) | 小模型可计数运算量(GFLOPs/样本) | 路由器延迟(毫秒/样本) | 路由器稠密运算量(GFLOPs/样本) | 大模型调用延迟(毫秒/样本) | 大模型完整生成估算(GFLOPs/调用) |",
|
| 296 |
+
"|---|---:|---:|---:|---:|---:|---:|",
|
| 297 |
+
]
|
| 298 |
+
for key, component in components.items():
|
| 299 |
+
lines.append(
|
| 300 |
+
f"| {component['label']} | {fmt(component['specialist_end_to_end_latency_ms'])} | {fmt(component['specialist_gflops_per_input'])} | {fmt(component['router_latency_ms'], 6)} | {fmt(component['router_gflops_per_input'], 6)} | {fmt(component['large_batch_wall_latency_ms_per_input'])} | {fmt(component['large_estimated_full_generation_gflops'])} |"
|
| 301 |
+
)
|
| 302 |
+
lines.extend([
|
| 303 |
+
"",
|
| 304 |
+
"说明:GFLOPs按乘加计2次运算,只覆盖计数器能识别的稠密乘法、卷积和注意力等算子;路由器的计数器返回0,因此按已加载检查点的线性层结构精确计算了稠密乘加量。大模型完整生成GFLOPs为“预填充GFLOPs + 平均生成长度 × 单Token解码GFLOPs”的估算,不把未计数算子冒充成完整硬件计数。",
|
| 305 |
+
"",
|
| 306 |
+
"## 4. 验证集锁定点在测试集上的结果",
|
| 307 |
+
"",
|
| 308 |
+
"下表的均值和标准差只在三个随机种子之间统计。成本使用对应的平均实际调用率;星号表示验证集选出的运行点。",
|
| 309 |
+
"",
|
| 310 |
+
"| 模型对 | 方法 | 测试性能 | 实际调用率 | 估算GFLOPs/输入 | 端到端延迟(毫秒/输入) |",
|
| 311 |
+
"|---|---|---:|---:|---:|---:|",
|
| 312 |
+
])
|
| 313 |
+
for key in components:
|
| 314 |
+
for method in ("both", "original", "confidence"):
|
| 315 |
+
row = selected_by_key[(key, method)]
|
| 316 |
+
metric = f"{100 * row['test_metric']:.2f}±{100 * row['test_metric_std']:.2f}%"
|
| 317 |
+
calls = f"{100 * row['actual_test_call_fraction']:.2f}±{100 * row['actual_test_call_fraction_std']:.2f}%"
|
| 318 |
+
lines.append(f"| {row['pair_label']} | {row['method_label']}* | {metric} | {calls} | {fmt(row['flops_gflops_per_input'])} | {fmt(row['latency_ms_per_input'])} |")
|
| 319 |
+
lines.extend([
|
| 320 |
+
"",
|
| 321 |
+
"### 两项同时改相对置信度路由",
|
| 322 |
+
"",
|
| 323 |
+
"| 模型对 | 性能变化 | 调用率变化 | 估算GFLOPs变化 | 实际延迟变化 |",
|
| 324 |
+
"|---|---:|---:|---:|---:|",
|
| 325 |
+
])
|
| 326 |
+
for key in components:
|
| 327 |
+
learned = selected_by_key[(key, "both")]
|
| 328 |
+
confidence = selected_by_key[(key, "confidence")]
|
| 329 |
+
metric_delta = 100 * (learned["test_metric"] - confidence["test_metric"])
|
| 330 |
+
call_delta = 100 * (learned["actual_test_call_fraction"] - confidence["actual_test_call_fraction"])
|
| 331 |
+
flop_delta = 100 * (learned["flops_gflops_per_input"] / confidence["flops_gflops_per_input"] - 1)
|
| 332 |
+
latency_delta = 100 * (learned["latency_ms_per_input"] / confidence["latency_ms_per_input"] - 1)
|
| 333 |
+
lines.append(f"| {learned['pair_label']} | {metric_delta:+.2f} 个百分点 | {call_delta:+.2f} 个百分点 | {flop_delta:+.1f}% | {latency_delta:+.1f}% |")
|
| 334 |
+
summary[key]["both_vs_confidence"] = {
|
| 335 |
+
"metric_delta_percentage_points": metric_delta,
|
| 336 |
+
"call_delta_percentage_points": call_delta,
|
| 337 |
+
"flops_relative_percent": flop_delta,
|
| 338 |
+
"latency_relative_percent": latency_delta,
|
| 339 |
+
}
|
| 340 |
+
lines.extend([
|
| 341 |
+
"",
|
| 342 |
+
"直白地说:两项同时改在这两个代表性模型对上都提高了测试性能,同时减少了实际调用,因此计算量和延迟也一起下降。原因不是路由器本身更快,而是它让大模型少被调用;路由器自身成本相对很小。",
|
| 343 |
+
"",
|
| 344 |
+
"## 5. 固定名义调用预算的代表点",
|
| 345 |
+
"",
|
| 346 |
+
"名义预算是验证阶段设定的阈值预算;阈值原样转到测试集后,实际调用率会有偏差。完整的99个阈值点和端点均在CSV中。",
|
| 347 |
+
"",
|
| 348 |
+
"| 模型对 | 名义预算 | 两项同时改:性能 / 实际调用 | 置信度:性能 / 实际调用 | 两项同时改:GFLOPs/输入 | 两项同时改:毫秒/输入 |",
|
| 349 |
+
"|---|---:|---:|---:|---:|---:|",
|
| 350 |
+
])
|
| 351 |
+
for key in components:
|
| 352 |
+
rows = {(row["nominal_budget"], row["method"]): row for row in fixed if row["pair_key"] == key}
|
| 353 |
+
for budget in (0.25, 0.50):
|
| 354 |
+
learned = rows[(budget, "both")]
|
| 355 |
+
confidence = rows[(budget, "confidence")]
|
| 356 |
+
lines.append(
|
| 357 |
+
f"| {components[key]['label']} | {100 * budget:.0f}% | {100 * learned['test_metric']:.2f}% / {100 * learned['actual_test_call_fraction']:.2f}% | {100 * confidence['test_metric']:.2f}% / {100 * confidence['actual_test_call_fraction']:.2f}% | {fmt(learned['flops_gflops_per_input'])} | {fmt(learned['latency_ms_per_input'])} |"
|
| 358 |
+
)
|
| 359 |
+
lines.extend([
|
| 360 |
+
"",
|
| 361 |
+
"## 6. 结果怎么理解",
|
| 362 |
+
"",
|
| 363 |
+
"1. gRefCOCO的InstanceVG + MiniCPM是这两组中相对置信度提升更明显的一组;Construction的YOLO26x + Qwen3.8也有稳定的正向变化。",
|
| 364 |
+
"2. 这里的成本是按真实测试调用率换算的平均每输入成本,不是把名义预算当成实际调用率。",
|
| 365 |
+
"3. 大模型延迟是A800、批量4下的均摊值,模型加载和预热没有计入;实际系统的并发度、批量大小和请求排队会改变墙钟延迟。",
|
| 366 |
+
"4. InstanceVG使用了远端已有的MMCV等价算子兼容路径,因为原生detrex扩展不可用;这不改变本次模型输出链路,但应在复现实验时保留该环境说明。",
|
| 367 |
+
"5. 本补充不修改正文,也不覆盖旧实验结果;它是针对两个代表性模型对的成本实测和成本—性能换算。",
|
| 368 |
+
"",
|
| 369 |
+
"## 7. 文件在哪里",
|
| 370 |
+
"",
|
| 371 |
+
f"- 原始A800测量记录:`{root.as_posix()}`目录下的8个JSON文件。",
|
| 372 |
+
f"- 完整成本—性能曲线数据:`{(root / 'full_cost_performance_curves.csv').as_posix()}`。",
|
| 373 |
+
f"- 固定预算代表点:`{(root / 'fixed_budget_cost_performance.csv').as_posix()}`。",
|
| 374 |
+
f"- FLOPs—性能图:`{(root / 'flops_performance.pdf').as_posix()}`;实际延迟—性能图:`{(root / 'latency_performance.pdf').as_posix()}`。",
|
| 375 |
+
f"- 可复现脚本:`{(root / 'make_cost_analysis_20260911.py').as_posix()}`;插入论文的LaTeX片段:`{(root / 'latex_includes.tex').as_posix()}`。",
|
| 376 |
+
])
|
| 377 |
+
return "\n".join(lines) + "\n"
|
| 378 |
+
|
| 379 |
+
|
| 380 |
+
def latex_includes() -> str:
|
| 381 |
+
return r"""% A800 cost--performance supplement (generated 2026-09-11)
|
| 382 |
+
\begin{figure*}[t]
|
| 383 |
+
\centering
|
| 384 |
+
\includegraphics[width=0.98\textwidth]{dse_results/method_improvement_20260911/cost_analysis_20260911/flops_performance.pdf}
|
| 385 |
+
\caption{A800 counted-FLOPs--performance curves for the two representative model pairs. Each point uses a validation-calibrated nominal budget transferred unchanged to test; the horizontal coordinate uses the resulting actual test call rate. The star marks the validation-selected operating point. Large-model full-generation FLOPs are estimated from measured prefill FLOPs, one-token decode FLOPs and the measured mean completion length; unsupported operations are not included.}
|
| 386 |
+
\label{fig:a800-flops-performance}
|
| 387 |
+
\end{figure*}
|
| 388 |
+
|
| 389 |
+
\begin{figure*}[t]
|
| 390 |
+
\centering
|
| 391 |
+
\includegraphics[width=0.98\textwidth]{dse_results/method_improvement_20260911/cost_analysis_20260911/latency_performance.pdf}
|
| 392 |
+
\caption{A800 end-to-end latency--performance curves. Specialist and router latency are always paid; the large-model term is weighted by the actual test call rate and uses the measured batch-wall latency per input. Model loading and warm-up are excluded.}
|
| 393 |
+
\label{fig:a800-latency-performance}
|
| 394 |
+
\end{figure*}
|
| 395 |
+
"""
|
| 396 |
+
|
| 397 |
+
|
| 398 |
+
def main() -> None:
|
| 399 |
+
parser = argparse.ArgumentParser()
|
| 400 |
+
parser.add_argument("--experiment-dir", type=Path, default=Path(__file__).resolve().parent.parent)
|
| 401 |
+
parser.add_argument("--output-dir", type=Path, default=Path(__file__).resolve().parent)
|
| 402 |
+
args = parser.parse_args()
|
| 403 |
+
root = args.output_dir.resolve()
|
| 404 |
+
experiment_dir = args.experiment_dir.resolve()
|
| 405 |
+
root.mkdir(parents=True, exist_ok=True)
|
| 406 |
+
|
| 407 |
+
architectures = read_json(root / "router_architecture.json")
|
| 408 |
+
architecture_keys = {item["pair_key"] for item in architectures}
|
| 409 |
+
if architecture_keys != set(PAIR_FILES):
|
| 410 |
+
raise ValueError("router architecture keys do not match selected pairs")
|
| 411 |
+
results = read_json(experiment_dir / "test_results.json")
|
| 412 |
+
components = build_cost_components(root, architectures)
|
| 413 |
+
all_curves = []
|
| 414 |
+
all_selected = []
|
| 415 |
+
for pair_key in PAIR_FILES:
|
| 416 |
+
groups = load_relevant_groups(results, pair_key)
|
| 417 |
+
all_curves.extend(curve_rows(groups, pair_key, components))
|
| 418 |
+
all_selected.extend(selected_rows(groups, pair_key, components))
|
| 419 |
+
all_fixed = fixed_budget_rows(all_curves)
|
| 420 |
+
|
| 421 |
+
write_csv(root / "full_cost_performance_curves.csv", all_curves)
|
| 422 |
+
write_csv(root / "fixed_budget_cost_performance.csv", all_fixed)
|
| 423 |
+
make_plot(all_curves, all_selected, root / "flops_performance.pdf", "flops_gflops_per_input", "Estimated GFLOPs / input", True)
|
| 424 |
+
make_plot(all_curves, all_selected, root / "latency_performance.pdf", "latency_ms_per_input", "End-to-end latency (ms / input)", False)
|
| 425 |
+
(root / "latex_includes.tex").write_text(latex_includes(), encoding="utf-8")
|
| 426 |
+
|
| 427 |
+
summary = {
|
| 428 |
+
"status": "complete",
|
| 429 |
+
"protocol": {
|
| 430 |
+
"test_results": str((experiment_dir / "test_results.json").resolve()),
|
| 431 |
+
"curves_use": "validation-calibrated thresholds transferred unchanged to test",
|
| 432 |
+
"cost_uses": "actual test call fraction",
|
| 433 |
+
"flop_counting": "multiply-add counted as 2 FLOPs; unsupported operations excluded",
|
| 434 |
+
"large_generation_flops": "mean prefill + mean completion tokens * one-token decode probe",
|
| 435 |
+
"seed_aggregation": "mean and population standard deviation across three seeds for learned/original methods",
|
| 436 |
+
},
|
| 437 |
+
"components": components,
|
| 438 |
+
"selected_points": all_selected,
|
| 439 |
+
"fixed_budget_points": all_fixed,
|
| 440 |
+
}
|
| 441 |
+
for key in components:
|
| 442 |
+
summary.setdefault("comparisons", {})
|
| 443 |
+
learned = next(row for row in all_selected if row["pair_key"] == key and row["method"] == "both")
|
| 444 |
+
confidence = next(row for row in all_selected if row["pair_key"] == key and row["method"] == "confidence")
|
| 445 |
+
summary["comparisons"][key] = {
|
| 446 |
+
"both_vs_confidence_metric_delta_percentage_points": 100 * (learned["test_metric"] - confidence["test_metric"]),
|
| 447 |
+
"both_vs_confidence_call_delta_percentage_points": 100 * (learned["actual_test_call_fraction"] - confidence["actual_test_call_fraction"]),
|
| 448 |
+
"both_vs_confidence_flops_relative_percent": 100 * (learned["flops_gflops_per_input"] / confidence["flops_gflops_per_input"] - 1),
|
| 449 |
+
"both_vs_confidence_latency_relative_percent": 100 * (learned["latency_ms_per_input"] / confidence["latency_ms_per_input"] - 1),
|
| 450 |
+
}
|
| 451 |
+
write_json(root / "cost_summary.json", summary)
|
| 452 |
+
(root / "成本性能补充报告.md").write_text(report_markdown(components, all_selected, all_fixed, root, summary["comparisons"]), encoding="utf-8")
|
| 453 |
+
print(json.dumps({"status": "complete", "output_dir": str(root), "curve_rows": len(all_curves), "fixed_rows": len(all_fixed)}, ensure_ascii=False))
|
| 454 |
+
|
| 455 |
+
|
| 456 |
+
if __name__ == "__main__":
|
| 457 |
+
main()
|
releases/20260913_lr/paper_data/cost_20260911/minicpm_grefcoco.json
ADDED
|
@@ -0,0 +1,385 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"model_kind": "MiniCPM",
|
| 4 |
+
"task": "grefcoco",
|
| 5 |
+
"expert": "instancevg",
|
| 6 |
+
"model_path": "/mnt/HithinkOmniSSD/user_workspace/caisihang/checkpoints/MiniCPM-V-4_5",
|
| 7 |
+
"device": "cuda:0",
|
| 8 |
+
"gpu_memory_utilization": 0.75,
|
| 9 |
+
"sample_count_requested": 32,
|
| 10 |
+
"warmup_count": 2,
|
| 11 |
+
"measured_count": 30,
|
| 12 |
+
"batch_size": 4,
|
| 13 |
+
"sample_ids": [
|
| 14 |
+
"testA-24562-67519",
|
| 15 |
+
"testB-37093-102233",
|
| 16 |
+
"testB-23966-65871",
|
| 17 |
+
"testB-66119-182290",
|
| 18 |
+
"testA-81450-224192",
|
| 19 |
+
"testB-33122-91196",
|
| 20 |
+
"testB-80267-220980",
|
| 21 |
+
"testA-47140-129791",
|
| 22 |
+
"testA-24439-67195",
|
| 23 |
+
"testA-54310-149436",
|
| 24 |
+
"testA-79491-218834",
|
| 25 |
+
"testA-10078-27741",
|
| 26 |
+
"testB-57040-156982",
|
| 27 |
+
"testA-26858-73835",
|
| 28 |
+
"testB-44430-122263",
|
| 29 |
+
"testA-60760-167399",
|
| 30 |
+
"testB-3182-8806",
|
| 31 |
+
"testB-947-2650",
|
| 32 |
+
"testA-73301-202004",
|
| 33 |
+
"testA-29200-80379",
|
| 34 |
+
"testB-42734-117684",
|
| 35 |
+
"testB-46676-128533",
|
| 36 |
+
"testB-8468-23291",
|
| 37 |
+
"testA-54557-150151",
|
| 38 |
+
"testA-58479-160960",
|
| 39 |
+
"testA-81251-223648",
|
| 40 |
+
"testA-5038-13944",
|
| 41 |
+
"testB-37760-104089",
|
| 42 |
+
"testA-62943-173467",
|
| 43 |
+
"testB-12467-34323",
|
| 44 |
+
"testA-80534-221720",
|
| 45 |
+
"testB-53121-146261"
|
| 46 |
+
],
|
| 47 |
+
"batches": [
|
| 48 |
+
{
|
| 49 |
+
"sample_ids": [
|
| 50 |
+
"testB-23966-65871",
|
| 51 |
+
"testB-66119-182290",
|
| 52 |
+
"testA-81450-224192",
|
| 53 |
+
"testB-33122-91196"
|
| 54 |
+
],
|
| 55 |
+
"batch_size": 4,
|
| 56 |
+
"generation_latency_seconds": 6.051569456933066,
|
| 57 |
+
"batch_wall_latency_seconds": 6.090657560969703,
|
| 58 |
+
"amortized_generation_ms_per_sample": 1512.8923642332666,
|
| 59 |
+
"amortized_batch_wall_ms_per_sample": 1522.6643902424257,
|
| 60 |
+
"outputs": [
|
| 61 |
+
{
|
| 62 |
+
"finish_reason": "length",
|
| 63 |
+
"stop_reason": null,
|
| 64 |
+
"prompt_tokens": 528,
|
| 65 |
+
"completion_tokens": 512,
|
| 66 |
+
"text_characters": 528
|
| 67 |
+
},
|
| 68 |
+
{
|
| 69 |
+
"finish_reason": "stop",
|
| 70 |
+
"stop_reason": null,
|
| 71 |
+
"prompt_tokens": 526,
|
| 72 |
+
"completion_tokens": 87,
|
| 73 |
+
"text_characters": 140
|
| 74 |
+
},
|
| 75 |
+
{
|
| 76 |
+
"finish_reason": "length",
|
| 77 |
+
"stop_reason": null,
|
| 78 |
+
"prompt_tokens": 523,
|
| 79 |
+
"completion_tokens": 512,
|
| 80 |
+
"text_characters": 528
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"finish_reason": "length",
|
| 84 |
+
"stop_reason": null,
|
| 85 |
+
"prompt_tokens": 527,
|
| 86 |
+
"completion_tokens": 512,
|
| 87 |
+
"text_characters": 528
|
| 88 |
+
}
|
| 89 |
+
]
|
| 90 |
+
},
|
| 91 |
+
{
|
| 92 |
+
"sample_ids": [
|
| 93 |
+
"testB-80267-220980",
|
| 94 |
+
"testA-47140-129791",
|
| 95 |
+
"testA-24439-67195",
|
| 96 |
+
"testA-54310-149436"
|
| 97 |
+
],
|
| 98 |
+
"batch_size": 4,
|
| 99 |
+
"generation_latency_seconds": 6.01708423695527,
|
| 100 |
+
"batch_wall_latency_seconds": 6.064165806979872,
|
| 101 |
+
"amortized_generation_ms_per_sample": 1504.2710592388175,
|
| 102 |
+
"amortized_batch_wall_ms_per_sample": 1516.041451744968,
|
| 103 |
+
"outputs": [
|
| 104 |
+
{
|
| 105 |
+
"finish_reason": "stop",
|
| 106 |
+
"stop_reason": null,
|
| 107 |
+
"prompt_tokens": 525,
|
| 108 |
+
"completion_tokens": 5,
|
| 109 |
+
"text_characters": 14
|
| 110 |
+
},
|
| 111 |
+
{
|
| 112 |
+
"finish_reason": "stop",
|
| 113 |
+
"stop_reason": null,
|
| 114 |
+
"prompt_tokens": 524,
|
| 115 |
+
"completion_tokens": 29,
|
| 116 |
+
"text_characters": 52
|
| 117 |
+
},
|
| 118 |
+
{
|
| 119 |
+
"finish_reason": "stop",
|
| 120 |
+
"stop_reason": null,
|
| 121 |
+
"prompt_tokens": 524,
|
| 122 |
+
"completion_tokens": 33,
|
| 123 |
+
"text_characters": 56
|
| 124 |
+
},
|
| 125 |
+
{
|
| 126 |
+
"finish_reason": "length",
|
| 127 |
+
"stop_reason": null,
|
| 128 |
+
"prompt_tokens": 522,
|
| 129 |
+
"completion_tokens": 512,
|
| 130 |
+
"text_characters": 528
|
| 131 |
+
}
|
| 132 |
+
]
|
| 133 |
+
},
|
| 134 |
+
{
|
| 135 |
+
"sample_ids": [
|
| 136 |
+
"testA-79491-218834",
|
| 137 |
+
"testA-10078-27741",
|
| 138 |
+
"testB-57040-156982",
|
| 139 |
+
"testA-26858-73835"
|
| 140 |
+
],
|
| 141 |
+
"batch_size": 4,
|
| 142 |
+
"generation_latency_seconds": 6.010764242964797,
|
| 143 |
+
"batch_wall_latency_seconds": 6.046397175989114,
|
| 144 |
+
"amortized_generation_ms_per_sample": 1502.6910607411992,
|
| 145 |
+
"amortized_batch_wall_ms_per_sample": 1511.5992939972784,
|
| 146 |
+
"outputs": [
|
| 147 |
+
{
|
| 148 |
+
"finish_reason": "stop",
|
| 149 |
+
"stop_reason": null,
|
| 150 |
+
"prompt_tokens": 524,
|
| 151 |
+
"completion_tokens": 62,
|
| 152 |
+
"text_characters": 100
|
| 153 |
+
},
|
| 154 |
+
{
|
| 155 |
+
"finish_reason": "stop",
|
| 156 |
+
"stop_reason": null,
|
| 157 |
+
"prompt_tokens": 529,
|
| 158 |
+
"completion_tokens": 87,
|
| 159 |
+
"text_characters": 140
|
| 160 |
+
},
|
| 161 |
+
{
|
| 162 |
+
"finish_reason": "length",
|
| 163 |
+
"stop_reason": null,
|
| 164 |
+
"prompt_tokens": 528,
|
| 165 |
+
"completion_tokens": 512,
|
| 166 |
+
"text_characters": 1690
|
| 167 |
+
},
|
| 168 |
+
{
|
| 169 |
+
"finish_reason": "stop",
|
| 170 |
+
"stop_reason": null,
|
| 171 |
+
"prompt_tokens": 521,
|
| 172 |
+
"completion_tokens": 5,
|
| 173 |
+
"text_characters": 14
|
| 174 |
+
}
|
| 175 |
+
]
|
| 176 |
+
},
|
| 177 |
+
{
|
| 178 |
+
"sample_ids": [
|
| 179 |
+
"testB-44430-122263",
|
| 180 |
+
"testA-60760-167399",
|
| 181 |
+
"testB-3182-8806",
|
| 182 |
+
"testB-947-2650"
|
| 183 |
+
],
|
| 184 |
+
"batch_size": 4,
|
| 185 |
+
"generation_latency_seconds": 2.593448459985666,
|
| 186 |
+
"batch_wall_latency_seconds": 2.6335477660177276,
|
| 187 |
+
"amortized_generation_ms_per_sample": 648.3621149964165,
|
| 188 |
+
"amortized_batch_wall_ms_per_sample": 658.3869415044319,
|
| 189 |
+
"outputs": [
|
| 190 |
+
{
|
| 191 |
+
"finish_reason": "stop",
|
| 192 |
+
"stop_reason": null,
|
| 193 |
+
"prompt_tokens": 525,
|
| 194 |
+
"completion_tokens": 6,
|
| 195 |
+
"text_characters": 14
|
| 196 |
+
},
|
| 197 |
+
{
|
| 198 |
+
"finish_reason": "stop",
|
| 199 |
+
"stop_reason": null,
|
| 200 |
+
"prompt_tokens": 392,
|
| 201 |
+
"completion_tokens": 33,
|
| 202 |
+
"text_characters": 56
|
| 203 |
+
},
|
| 204 |
+
{
|
| 205 |
+
"finish_reason": "stop",
|
| 206 |
+
"stop_reason": null,
|
| 207 |
+
"prompt_tokens": 392,
|
| 208 |
+
"completion_tokens": 33,
|
| 209 |
+
"text_characters": 56
|
| 210 |
+
},
|
| 211 |
+
{
|
| 212 |
+
"finish_reason": "stop",
|
| 213 |
+
"stop_reason": null,
|
| 214 |
+
"prompt_tokens": 528,
|
| 215 |
+
"completion_tokens": 208,
|
| 216 |
+
"text_characters": 321
|
| 217 |
+
}
|
| 218 |
+
]
|
| 219 |
+
},
|
| 220 |
+
{
|
| 221 |
+
"sample_ids": [
|
| 222 |
+
"testA-73301-202004",
|
| 223 |
+
"testA-29200-80379",
|
| 224 |
+
"testB-42734-117684",
|
| 225 |
+
"testB-46676-128533"
|
| 226 |
+
],
|
| 227 |
+
"batch_size": 4,
|
| 228 |
+
"generation_latency_seconds": 6.017314173979685,
|
| 229 |
+
"batch_wall_latency_seconds": 6.065163187915459,
|
| 230 |
+
"amortized_generation_ms_per_sample": 1504.3285434949212,
|
| 231 |
+
"amortized_batch_wall_ms_per_sample": 1516.2907969788648,
|
| 232 |
+
"outputs": [
|
| 233 |
+
{
|
| 234 |
+
"finish_reason": "stop",
|
| 235 |
+
"stop_reason": null,
|
| 236 |
+
"prompt_tokens": 525,
|
| 237 |
+
"completion_tokens": 61,
|
| 238 |
+
"text_characters": 99
|
| 239 |
+
},
|
| 240 |
+
{
|
| 241 |
+
"finish_reason": "stop",
|
| 242 |
+
"stop_reason": null,
|
| 243 |
+
"prompt_tokens": 520,
|
| 244 |
+
"completion_tokens": 5,
|
| 245 |
+
"text_characters": 14
|
| 246 |
+
},
|
| 247 |
+
{
|
| 248 |
+
"finish_reason": "length",
|
| 249 |
+
"stop_reason": null,
|
| 250 |
+
"prompt_tokens": 523,
|
| 251 |
+
"completion_tokens": 512,
|
| 252 |
+
"text_characters": 528
|
| 253 |
+
},
|
| 254 |
+
{
|
| 255 |
+
"finish_reason": "stop",
|
| 256 |
+
"stop_reason": null,
|
| 257 |
+
"prompt_tokens": 528,
|
| 258 |
+
"completion_tokens": 5,
|
| 259 |
+
"text_characters": 14
|
| 260 |
+
}
|
| 261 |
+
]
|
| 262 |
+
},
|
| 263 |
+
{
|
| 264 |
+
"sample_ids": [
|
| 265 |
+
"testB-8468-23291",
|
| 266 |
+
"testA-54557-150151",
|
| 267 |
+
"testA-58479-160960",
|
| 268 |
+
"testA-81251-223648"
|
| 269 |
+
],
|
| 270 |
+
"batch_size": 4,
|
| 271 |
+
"generation_latency_seconds": 1.6802827849751338,
|
| 272 |
+
"batch_wall_latency_seconds": 1.716626849025488,
|
| 273 |
+
"amortized_generation_ms_per_sample": 420.07069624378346,
|
| 274 |
+
"amortized_batch_wall_ms_per_sample": 429.156712256372,
|
| 275 |
+
"outputs": [
|
| 276 |
+
{
|
| 277 |
+
"finish_reason": "stop",
|
| 278 |
+
"stop_reason": null,
|
| 279 |
+
"prompt_tokens": 539,
|
| 280 |
+
"completion_tokens": 62,
|
| 281 |
+
"text_characters": 100
|
| 282 |
+
},
|
| 283 |
+
{
|
| 284 |
+
"finish_reason": "stop",
|
| 285 |
+
"stop_reason": null,
|
| 286 |
+
"prompt_tokens": 522,
|
| 287 |
+
"completion_tokens": 30,
|
| 288 |
+
"text_characters": 53
|
| 289 |
+
},
|
| 290 |
+
{
|
| 291 |
+
"finish_reason": "stop",
|
| 292 |
+
"stop_reason": null,
|
| 293 |
+
"prompt_tokens": 529,
|
| 294 |
+
"completion_tokens": 6,
|
| 295 |
+
"text_characters": 14
|
| 296 |
+
},
|
| 297 |
+
{
|
| 298 |
+
"finish_reason": "stop",
|
| 299 |
+
"stop_reason": null,
|
| 300 |
+
"prompt_tokens": 525,
|
| 301 |
+
"completion_tokens": 122,
|
| 302 |
+
"text_characters": 190
|
| 303 |
+
}
|
| 304 |
+
]
|
| 305 |
+
},
|
| 306 |
+
{
|
| 307 |
+
"sample_ids": [
|
| 308 |
+
"testA-5038-13944",
|
| 309 |
+
"testB-37760-104089",
|
| 310 |
+
"testA-62943-173467",
|
| 311 |
+
"testB-12467-34323"
|
| 312 |
+
],
|
| 313 |
+
"batch_size": 4,
|
| 314 |
+
"generation_latency_seconds": 5.998426596052013,
|
| 315 |
+
"batch_wall_latency_seconds": 6.038230134989135,
|
| 316 |
+
"amortized_generation_ms_per_sample": 1499.6066490130033,
|
| 317 |
+
"amortized_batch_wall_ms_per_sample": 1509.5575337472837,
|
| 318 |
+
"outputs": [
|
| 319 |
+
{
|
| 320 |
+
"finish_reason": "stop",
|
| 321 |
+
"stop_reason": null,
|
| 322 |
+
"prompt_tokens": 522,
|
| 323 |
+
"completion_tokens": 39,
|
| 324 |
+
"text_characters": 64
|
| 325 |
+
},
|
| 326 |
+
{
|
| 327 |
+
"finish_reason": "length",
|
| 328 |
+
"stop_reason": null,
|
| 329 |
+
"prompt_tokens": 536,
|
| 330 |
+
"completion_tokens": 512,
|
| 331 |
+
"text_characters": 528
|
| 332 |
+
},
|
| 333 |
+
{
|
| 334 |
+
"finish_reason": "stop",
|
| 335 |
+
"stop_reason": null,
|
| 336 |
+
"prompt_tokens": 538,
|
| 337 |
+
"completion_tokens": 33,
|
| 338 |
+
"text_characters": 56
|
| 339 |
+
},
|
| 340 |
+
{
|
| 341 |
+
"finish_reason": "stop",
|
| 342 |
+
"stop_reason": null,
|
| 343 |
+
"prompt_tokens": 522,
|
| 344 |
+
"completion_tokens": 5,
|
| 345 |
+
"text_characters": 14
|
| 346 |
+
}
|
| 347 |
+
]
|
| 348 |
+
},
|
| 349 |
+
{
|
| 350 |
+
"sample_ids": [
|
| 351 |
+
"testA-80534-221720",
|
| 352 |
+
"testB-53121-146261"
|
| 353 |
+
],
|
| 354 |
+
"batch_size": 2,
|
| 355 |
+
"generation_latency_seconds": 5.806203712942079,
|
| 356 |
+
"batch_wall_latency_seconds": 5.831518966006115,
|
| 357 |
+
"amortized_generation_ms_per_sample": 2903.1018564710394,
|
| 358 |
+
"amortized_batch_wall_ms_per_sample": 2915.7594830030575,
|
| 359 |
+
"outputs": [
|
| 360 |
+
{
|
| 361 |
+
"finish_reason": "length",
|
| 362 |
+
"stop_reason": null,
|
| 363 |
+
"prompt_tokens": 390,
|
| 364 |
+
"completion_tokens": 512,
|
| 365 |
+
"text_characters": 528
|
| 366 |
+
},
|
| 367 |
+
{
|
| 368 |
+
"finish_reason": "stop",
|
| 369 |
+
"stop_reason": null,
|
| 370 |
+
"prompt_tokens": 523,
|
| 371 |
+
"completion_tokens": 33,
|
| 372 |
+
"text_characters": 56
|
| 373 |
+
}
|
| 374 |
+
]
|
| 375 |
+
}
|
| 376 |
+
],
|
| 377 |
+
"mean_generation_ms_per_sample": 1436.915543054056,
|
| 378 |
+
"median_generation_ms_per_sample": 1503.4810599900084,
|
| 379 |
+
"mean_batch_wall_ms_per_sample": 1447.4320754343353,
|
| 380 |
+
"median_batch_wall_ms_per_sample": 1513.8203728711233,
|
| 381 |
+
"mean_prompt_tokens": 512.6666666666666,
|
| 382 |
+
"mean_completion_tokens": 169.5,
|
| 383 |
+
"latency_scope": "A800 vLLM-0.18.0 native generation latency with the frozen task prompt and structured-output schema; model loading and warmup excluded. Batch wall time includes the real request/image preparation path.",
|
| 384 |
+
"flops_scope": "This file measures latency only. Counted FLOPs are measured separately by the HF reference probe and are not claimed to be complete vLLM kernel FLOPs."
|
| 385 |
+
}
|
releases/20260913_lr/paper_data/cost_20260911/minicpm_grefcoco_flops.json
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"model_kind": "MiniCPM",
|
| 4 |
+
"task": "grefcoco",
|
| 5 |
+
"model_path": "/mnt/HithinkOmniSSD/user_workspace/caisihang/checkpoints/MiniCPM-V-4_5",
|
| 6 |
+
"sample_count": 4,
|
| 7 |
+
"records": [
|
| 8 |
+
{
|
| 9 |
+
"sample_id": "testB-37890-104442",
|
| 10 |
+
"prompt_tokens_in_reference_tensor": 528,
|
| 11 |
+
"input_shapes": {
|
| 12 |
+
"input_ids": [
|
| 13 |
+
1,
|
| 14 |
+
528
|
| 15 |
+
],
|
| 16 |
+
"attention_mask": [
|
| 17 |
+
1,
|
| 18 |
+
528
|
| 19 |
+
]
|
| 20 |
+
},
|
| 21 |
+
"prefill_flops_counted": 7500407080960,
|
| 22 |
+
"decode_one_token_flops_counted": 15446671488,
|
| 23 |
+
"prefill_gflops_counted": 7500.40708096,
|
| 24 |
+
"decode_one_token_gflops_counted": 15.446671488,
|
| 25 |
+
"prefill_operators": {
|
| 26 |
+
"aten.bmm": 164433561600,
|
| 27 |
+
"aten.mm": 7335973519360
|
| 28 |
+
},
|
| 29 |
+
"decode_operators": {
|
| 30 |
+
"aten.bmm": 312017024,
|
| 31 |
+
"aten.mm": 15134654464
|
| 32 |
+
}
|
| 33 |
+
},
|
| 34 |
+
{
|
| 35 |
+
"sample_id": "testA-73305-202016",
|
| 36 |
+
"prompt_tokens_in_reference_tensor": 525,
|
| 37 |
+
"input_shapes": {
|
| 38 |
+
"input_ids": [
|
| 39 |
+
1,
|
| 40 |
+
525
|
| 41 |
+
],
|
| 42 |
+
"attention_mask": [
|
| 43 |
+
1,
|
| 44 |
+
525
|
| 45 |
+
]
|
| 46 |
+
},
|
| 47 |
+
"prefill_flops_counted": 7456869222016,
|
| 48 |
+
"decode_one_token_flops_counted": 15444902016,
|
| 49 |
+
"prefill_gflops_counted": 7456.869222016,
|
| 50 |
+
"decode_one_token_gflops_counted": 15.444902016,
|
| 51 |
+
"prefill_operators": {
|
| 52 |
+
"aten.bmm": 162570307200,
|
| 53 |
+
"aten.mm": 7294298914816
|
| 54 |
+
},
|
| 55 |
+
"decode_operators": {
|
| 56 |
+
"aten.bmm": 310247552,
|
| 57 |
+
"aten.mm": 15134654464
|
| 58 |
+
}
|
| 59 |
+
},
|
| 60 |
+
{
|
| 61 |
+
"sample_id": "testB-24036-66087",
|
| 62 |
+
"prompt_tokens_in_reference_tensor": 522,
|
| 63 |
+
"input_shapes": {
|
| 64 |
+
"input_ids": [
|
| 65 |
+
1,
|
| 66 |
+
522
|
| 67 |
+
],
|
| 68 |
+
"attention_mask": [
|
| 69 |
+
1,
|
| 70 |
+
522
|
| 71 |
+
]
|
| 72 |
+
},
|
| 73 |
+
"prefill_flops_counted": 7413341979904,
|
| 74 |
+
"decode_one_token_flops_counted": 15443132544,
|
| 75 |
+
"prefill_gflops_counted": 7413.341979904,
|
| 76 |
+
"decode_one_token_gflops_counted": 15.443132544,
|
| 77 |
+
"prefill_operators": {
|
| 78 |
+
"aten.bmm": 160717669632,
|
| 79 |
+
"aten.mm": 7252624310272
|
| 80 |
+
},
|
| 81 |
+
"decode_operators": {
|
| 82 |
+
"aten.bmm": 308478080,
|
| 83 |
+
"aten.mm": 15134654464
|
| 84 |
+
}
|
| 85 |
+
},
|
| 86 |
+
{
|
| 87 |
+
"sample_id": "testA-5038-13945",
|
| 88 |
+
"prompt_tokens_in_reference_tensor": 528,
|
| 89 |
+
"input_shapes": {
|
| 90 |
+
"input_ids": [
|
| 91 |
+
1,
|
| 92 |
+
528
|
| 93 |
+
],
|
| 94 |
+
"attention_mask": [
|
| 95 |
+
1,
|
| 96 |
+
528
|
| 97 |
+
]
|
| 98 |
+
},
|
| 99 |
+
"prefill_flops_counted": 7500407080960,
|
| 100 |
+
"decode_one_token_flops_counted": 15446671488,
|
| 101 |
+
"prefill_gflops_counted": 7500.40708096,
|
| 102 |
+
"decode_one_token_gflops_counted": 15.446671488,
|
| 103 |
+
"prefill_operators": {
|
| 104 |
+
"aten.bmm": 164433561600,
|
| 105 |
+
"aten.mm": 7335973519360
|
| 106 |
+
},
|
| 107 |
+
"decode_operators": {
|
| 108 |
+
"aten.bmm": 312017024,
|
| 109 |
+
"aten.mm": 15134654464
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
],
|
| 113 |
+
"mean_prefill_gflops_counted": 7467.756340960001,
|
| 114 |
+
"mean_decode_one_token_gflops_counted": 15.445344384,
|
| 115 |
+
"scope": "HF reference execution on the A800 with real task inputs; dense products/convolutions/supported attention counted with multiply-add=2 FLOPs. vLLM custom kernels, elementwise/normalization/nonlinearities and other unsupported operations are not complete.",
|
| 116 |
+
"answer_generation": "not performed; one known-token decode probe only"
|
| 117 |
+
}
|
releases/20260913_lr/paper_data/cost_20260911/qwen_construction.json
ADDED
|
@@ -0,0 +1,385 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"model_kind": "Qwen3.8",
|
| 4 |
+
"task": "construction",
|
| 5 |
+
"expert": "yolo26x",
|
| 6 |
+
"model_path": "/mnt/HithinkOmniSSD/user_workspace/caisihang/checkpoints/Qwen3.8-27B-FP8",
|
| 7 |
+
"device": "cuda:0",
|
| 8 |
+
"gpu_memory_utilization": 0.75,
|
| 9 |
+
"sample_count_requested": 32,
|
| 10 |
+
"warmup_count": 2,
|
| 11 |
+
"measured_count": 30,
|
| 12 |
+
"batch_size": 4,
|
| 13 |
+
"sample_ids": [
|
| 14 |
+
"0000772",
|
| 15 |
+
"0003400",
|
| 16 |
+
"0003077",
|
| 17 |
+
"0004213",
|
| 18 |
+
"0002162",
|
| 19 |
+
"0003303",
|
| 20 |
+
"0004535",
|
| 21 |
+
"0001299",
|
| 22 |
+
"0000762",
|
| 23 |
+
"0001453",
|
| 24 |
+
"0002112",
|
| 25 |
+
"0000382",
|
| 26 |
+
"0003936",
|
| 27 |
+
"0000833",
|
| 28 |
+
"0003551",
|
| 29 |
+
"0001683",
|
| 30 |
+
"0002571",
|
| 31 |
+
"0002523",
|
| 32 |
+
"0001939",
|
| 33 |
+
"0000904",
|
| 34 |
+
"0003516",
|
| 35 |
+
"0003633",
|
| 36 |
+
"0002690",
|
| 37 |
+
"0001464",
|
| 38 |
+
"0001595",
|
| 39 |
+
"0002168",
|
| 40 |
+
"0000183",
|
| 41 |
+
"0003404",
|
| 42 |
+
"0001727",
|
| 43 |
+
"0002796",
|
| 44 |
+
"0002153",
|
| 45 |
+
"0003848"
|
| 46 |
+
],
|
| 47 |
+
"batches": [
|
| 48 |
+
{
|
| 49 |
+
"sample_ids": [
|
| 50 |
+
"0003077",
|
| 51 |
+
"0004213",
|
| 52 |
+
"0002162",
|
| 53 |
+
"0003303"
|
| 54 |
+
],
|
| 55 |
+
"batch_size": 4,
|
| 56 |
+
"generation_latency_seconds": 4.881686039967462,
|
| 57 |
+
"batch_wall_latency_seconds": 4.988335099071264,
|
| 58 |
+
"amortized_generation_ms_per_sample": 1220.4215099918656,
|
| 59 |
+
"amortized_batch_wall_ms_per_sample": 1247.083774767816,
|
| 60 |
+
"outputs": [
|
| 61 |
+
{
|
| 62 |
+
"finish_reason": "stop",
|
| 63 |
+
"stop_reason": null,
|
| 64 |
+
"prompt_tokens": 1705,
|
| 65 |
+
"completion_tokens": 110,
|
| 66 |
+
"text_characters": 345
|
| 67 |
+
},
|
| 68 |
+
{
|
| 69 |
+
"finish_reason": "stop",
|
| 70 |
+
"stop_reason": null,
|
| 71 |
+
"prompt_tokens": 1705,
|
| 72 |
+
"completion_tokens": 110,
|
| 73 |
+
"text_characters": 345
|
| 74 |
+
},
|
| 75 |
+
{
|
| 76 |
+
"finish_reason": "stop",
|
| 77 |
+
"stop_reason": null,
|
| 78 |
+
"prompt_tokens": 1705,
|
| 79 |
+
"completion_tokens": 110,
|
| 80 |
+
"text_characters": 345
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"finish_reason": "stop",
|
| 84 |
+
"stop_reason": null,
|
| 85 |
+
"prompt_tokens": 1705,
|
| 86 |
+
"completion_tokens": 110,
|
| 87 |
+
"text_characters": 345
|
| 88 |
+
}
|
| 89 |
+
]
|
| 90 |
+
},
|
| 91 |
+
{
|
| 92 |
+
"sample_ids": [
|
| 93 |
+
"0004535",
|
| 94 |
+
"0001299",
|
| 95 |
+
"0000762",
|
| 96 |
+
"0001453"
|
| 97 |
+
],
|
| 98 |
+
"batch_size": 4,
|
| 99 |
+
"generation_latency_seconds": 7.0890667490893975,
|
| 100 |
+
"batch_wall_latency_seconds": 7.216836072970182,
|
| 101 |
+
"amortized_generation_ms_per_sample": 1772.2666872723494,
|
| 102 |
+
"amortized_batch_wall_ms_per_sample": 1804.2090182425454,
|
| 103 |
+
"outputs": [
|
| 104 |
+
{
|
| 105 |
+
"finish_reason": "stop",
|
| 106 |
+
"stop_reason": null,
|
| 107 |
+
"prompt_tokens": 1705,
|
| 108 |
+
"completion_tokens": 110,
|
| 109 |
+
"text_characters": 345
|
| 110 |
+
},
|
| 111 |
+
{
|
| 112 |
+
"finish_reason": "stop",
|
| 113 |
+
"stop_reason": null,
|
| 114 |
+
"prompt_tokens": 1694,
|
| 115 |
+
"completion_tokens": 110,
|
| 116 |
+
"text_characters": 345
|
| 117 |
+
},
|
| 118 |
+
{
|
| 119 |
+
"finish_reason": "stop",
|
| 120 |
+
"stop_reason": null,
|
| 121 |
+
"prompt_tokens": 1531,
|
| 122 |
+
"completion_tokens": 144,
|
| 123 |
+
"text_characters": 435
|
| 124 |
+
},
|
| 125 |
+
{
|
| 126 |
+
"finish_reason": "stop",
|
| 127 |
+
"stop_reason": null,
|
| 128 |
+
"prompt_tokens": 1531,
|
| 129 |
+
"completion_tokens": 226,
|
| 130 |
+
"text_characters": 616
|
| 131 |
+
}
|
| 132 |
+
]
|
| 133 |
+
},
|
| 134 |
+
{
|
| 135 |
+
"sample_ids": [
|
| 136 |
+
"0002112",
|
| 137 |
+
"0000382",
|
| 138 |
+
"0003936",
|
| 139 |
+
"0000833"
|
| 140 |
+
],
|
| 141 |
+
"batch_size": 4,
|
| 142 |
+
"generation_latency_seconds": 5.4999597480054945,
|
| 143 |
+
"batch_wall_latency_seconds": 5.588348102988675,
|
| 144 |
+
"amortized_generation_ms_per_sample": 1374.9899370013736,
|
| 145 |
+
"amortized_batch_wall_ms_per_sample": 1397.0870257471688,
|
| 146 |
+
"outputs": [
|
| 147 |
+
{
|
| 148 |
+
"finish_reason": "stop",
|
| 149 |
+
"stop_reason": null,
|
| 150 |
+
"prompt_tokens": 1405,
|
| 151 |
+
"completion_tokens": 153,
|
| 152 |
+
"text_characters": 480
|
| 153 |
+
},
|
| 154 |
+
{
|
| 155 |
+
"finish_reason": "stop",
|
| 156 |
+
"stop_reason": null,
|
| 157 |
+
"prompt_tokens": 1705,
|
| 158 |
+
"completion_tokens": 110,
|
| 159 |
+
"text_characters": 345
|
| 160 |
+
},
|
| 161 |
+
{
|
| 162 |
+
"finish_reason": "stop",
|
| 163 |
+
"stop_reason": null,
|
| 164 |
+
"prompt_tokens": 1531,
|
| 165 |
+
"completion_tokens": 110,
|
| 166 |
+
"text_characters": 345
|
| 167 |
+
},
|
| 168 |
+
{
|
| 169 |
+
"finish_reason": "stop",
|
| 170 |
+
"stop_reason": null,
|
| 171 |
+
"prompt_tokens": 1683,
|
| 172 |
+
"completion_tokens": 139,
|
| 173 |
+
"text_characters": 409
|
| 174 |
+
}
|
| 175 |
+
]
|
| 176 |
+
},
|
| 177 |
+
{
|
| 178 |
+
"sample_ids": [
|
| 179 |
+
"0003551",
|
| 180 |
+
"0001683",
|
| 181 |
+
"0002571",
|
| 182 |
+
"0002523"
|
| 183 |
+
],
|
| 184 |
+
"batch_size": 4,
|
| 185 |
+
"generation_latency_seconds": 4.797023626044393,
|
| 186 |
+
"batch_wall_latency_seconds": 4.88577020005323,
|
| 187 |
+
"amortized_generation_ms_per_sample": 1199.2559065110981,
|
| 188 |
+
"amortized_batch_wall_ms_per_sample": 1221.4425500133075,
|
| 189 |
+
"outputs": [
|
| 190 |
+
{
|
| 191 |
+
"finish_reason": "stop",
|
| 192 |
+
"stop_reason": null,
|
| 193 |
+
"prompt_tokens": 1705,
|
| 194 |
+
"completion_tokens": 110,
|
| 195 |
+
"text_characters": 345
|
| 196 |
+
},
|
| 197 |
+
{
|
| 198 |
+
"finish_reason": "stop",
|
| 199 |
+
"stop_reason": null,
|
| 200 |
+
"prompt_tokens": 1705,
|
| 201 |
+
"completion_tokens": 110,
|
| 202 |
+
"text_characters": 345
|
| 203 |
+
},
|
| 204 |
+
{
|
| 205 |
+
"finish_reason": "stop",
|
| 206 |
+
"stop_reason": null,
|
| 207 |
+
"prompt_tokens": 1694,
|
| 208 |
+
"completion_tokens": 110,
|
| 209 |
+
"text_characters": 345
|
| 210 |
+
},
|
| 211 |
+
{
|
| 212 |
+
"finish_reason": "stop",
|
| 213 |
+
"stop_reason": null,
|
| 214 |
+
"prompt_tokens": 1705,
|
| 215 |
+
"completion_tokens": 110,
|
| 216 |
+
"text_characters": 345
|
| 217 |
+
}
|
| 218 |
+
]
|
| 219 |
+
},
|
| 220 |
+
{
|
| 221 |
+
"sample_ids": [
|
| 222 |
+
"0001939",
|
| 223 |
+
"0000904",
|
| 224 |
+
"0003516",
|
| 225 |
+
"0003633"
|
| 226 |
+
],
|
| 227 |
+
"batch_size": 4,
|
| 228 |
+
"generation_latency_seconds": 6.3565617019776255,
|
| 229 |
+
"batch_wall_latency_seconds": 6.429374794010073,
|
| 230 |
+
"amortized_generation_ms_per_sample": 1589.1404254944064,
|
| 231 |
+
"amortized_batch_wall_ms_per_sample": 1607.3436985025182,
|
| 232 |
+
"outputs": [
|
| 233 |
+
{
|
| 234 |
+
"finish_reason": "stop",
|
| 235 |
+
"stop_reason": null,
|
| 236 |
+
"prompt_tokens": 1705,
|
| 237 |
+
"completion_tokens": 110,
|
| 238 |
+
"text_characters": 345
|
| 239 |
+
},
|
| 240 |
+
{
|
| 241 |
+
"finish_reason": "stop",
|
| 242 |
+
"stop_reason": null,
|
| 243 |
+
"prompt_tokens": 1705,
|
| 244 |
+
"completion_tokens": 147,
|
| 245 |
+
"text_characters": 465
|
| 246 |
+
},
|
| 247 |
+
{
|
| 248 |
+
"finish_reason": "stop",
|
| 249 |
+
"stop_reason": null,
|
| 250 |
+
"prompt_tokens": 1683,
|
| 251 |
+
"completion_tokens": 185,
|
| 252 |
+
"text_characters": 574
|
| 253 |
+
},
|
| 254 |
+
{
|
| 255 |
+
"finish_reason": "stop",
|
| 256 |
+
"stop_reason": null,
|
| 257 |
+
"prompt_tokens": 1405,
|
| 258 |
+
"completion_tokens": 141,
|
| 259 |
+
"text_characters": 357
|
| 260 |
+
}
|
| 261 |
+
]
|
| 262 |
+
},
|
| 263 |
+
{
|
| 264 |
+
"sample_ids": [
|
| 265 |
+
"0002690",
|
| 266 |
+
"0001464",
|
| 267 |
+
"0001595",
|
| 268 |
+
"0002168"
|
| 269 |
+
],
|
| 270 |
+
"batch_size": 4,
|
| 271 |
+
"generation_latency_seconds": 11.480005273944698,
|
| 272 |
+
"batch_wall_latency_seconds": 11.721151808975264,
|
| 273 |
+
"amortized_generation_ms_per_sample": 2870.0013184861746,
|
| 274 |
+
"amortized_batch_wall_ms_per_sample": 2930.287952243816,
|
| 275 |
+
"outputs": [
|
| 276 |
+
{
|
| 277 |
+
"finish_reason": "stop",
|
| 278 |
+
"stop_reason": null,
|
| 279 |
+
"prompt_tokens": 1705,
|
| 280 |
+
"completion_tokens": 110,
|
| 281 |
+
"text_characters": 345
|
| 282 |
+
},
|
| 283 |
+
{
|
| 284 |
+
"finish_reason": "stop",
|
| 285 |
+
"stop_reason": null,
|
| 286 |
+
"prompt_tokens": 1705,
|
| 287 |
+
"completion_tokens": 110,
|
| 288 |
+
"text_characters": 345
|
| 289 |
+
},
|
| 290 |
+
{
|
| 291 |
+
"finish_reason": "stop",
|
| 292 |
+
"stop_reason": null,
|
| 293 |
+
"prompt_tokens": 1705,
|
| 294 |
+
"completion_tokens": 177,
|
| 295 |
+
"text_characters": 545
|
| 296 |
+
},
|
| 297 |
+
{
|
| 298 |
+
"finish_reason": "stop",
|
| 299 |
+
"stop_reason": null,
|
| 300 |
+
"prompt_tokens": 1705,
|
| 301 |
+
"completion_tokens": 155,
|
| 302 |
+
"text_characters": 481
|
| 303 |
+
}
|
| 304 |
+
]
|
| 305 |
+
},
|
| 306 |
+
{
|
| 307 |
+
"sample_ids": [
|
| 308 |
+
"0000183",
|
| 309 |
+
"0003404",
|
| 310 |
+
"0001727",
|
| 311 |
+
"0002796"
|
| 312 |
+
],
|
| 313 |
+
"batch_size": 4,
|
| 314 |
+
"generation_latency_seconds": 13.841306244954467,
|
| 315 |
+
"batch_wall_latency_seconds": 13.994719217997044,
|
| 316 |
+
"amortized_generation_ms_per_sample": 3460.3265612386167,
|
| 317 |
+
"amortized_batch_wall_ms_per_sample": 3498.679804499261,
|
| 318 |
+
"outputs": [
|
| 319 |
+
{
|
| 320 |
+
"finish_reason": "stop",
|
| 321 |
+
"stop_reason": null,
|
| 322 |
+
"prompt_tokens": 1705,
|
| 323 |
+
"completion_tokens": 157,
|
| 324 |
+
"text_characters": 510
|
| 325 |
+
},
|
| 326 |
+
{
|
| 327 |
+
"finish_reason": "stop",
|
| 328 |
+
"stop_reason": null,
|
| 329 |
+
"prompt_tokens": 1705,
|
| 330 |
+
"completion_tokens": 150,
|
| 331 |
+
"text_characters": 462
|
| 332 |
+
},
|
| 333 |
+
{
|
| 334 |
+
"finish_reason": "stop",
|
| 335 |
+
"stop_reason": null,
|
| 336 |
+
"prompt_tokens": 1531,
|
| 337 |
+
"completion_tokens": 169,
|
| 338 |
+
"text_characters": 508
|
| 339 |
+
},
|
| 340 |
+
{
|
| 341 |
+
"finish_reason": "stop",
|
| 342 |
+
"stop_reason": null,
|
| 343 |
+
"prompt_tokens": 1705,
|
| 344 |
+
"completion_tokens": 187,
|
| 345 |
+
"text_characters": 511
|
| 346 |
+
}
|
| 347 |
+
]
|
| 348 |
+
},
|
| 349 |
+
{
|
| 350 |
+
"sample_ids": [
|
| 351 |
+
"0002153",
|
| 352 |
+
"0003848"
|
| 353 |
+
],
|
| 354 |
+
"batch_size": 2,
|
| 355 |
+
"generation_latency_seconds": 14.003285999991931,
|
| 356 |
+
"batch_wall_latency_seconds": 14.185666976962239,
|
| 357 |
+
"amortized_generation_ms_per_sample": 7001.6429999959655,
|
| 358 |
+
"amortized_batch_wall_ms_per_sample": 7092.833488481119,
|
| 359 |
+
"outputs": [
|
| 360 |
+
{
|
| 361 |
+
"finish_reason": "stop",
|
| 362 |
+
"stop_reason": null,
|
| 363 |
+
"prompt_tokens": 1705,
|
| 364 |
+
"completion_tokens": 221,
|
| 365 |
+
"text_characters": 616
|
| 366 |
+
},
|
| 367 |
+
{
|
| 368 |
+
"finish_reason": "stop",
|
| 369 |
+
"stop_reason": null,
|
| 370 |
+
"prompt_tokens": 1705,
|
| 371 |
+
"completion_tokens": 110,
|
| 372 |
+
"text_characters": 345
|
| 373 |
+
}
|
| 374 |
+
]
|
| 375 |
+
}
|
| 376 |
+
],
|
| 377 |
+
"mean_generation_ms_per_sample": 2561.0056682489812,
|
| 378 |
+
"median_generation_ms_per_sample": 1680.7035563833779,
|
| 379 |
+
"mean_batch_wall_ms_per_sample": 2599.870914062194,
|
| 380 |
+
"median_batch_wall_ms_per_sample": 1705.7763583725318,
|
| 381 |
+
"mean_prompt_tokens": 1659.6,
|
| 382 |
+
"mean_completion_tokens": 137.03333333333333,
|
| 383 |
+
"latency_scope": "A800 vLLM-0.18.0 native generation latency with the frozen task prompt and structured-output schema; model loading and warmup excluded. Batch wall time includes the real request/image preparation path.",
|
| 384 |
+
"flops_scope": "This file measures latency only. Counted FLOPs are measured separately by the HF reference probe and are not claimed to be complete vLLM kernel FLOPs."
|
| 385 |
+
}
|
releases/20260913_lr/paper_data/cost_20260911/qwen_construction_flops.json
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"model_kind": "Qwen3.8",
|
| 4 |
+
"task": "construction",
|
| 5 |
+
"model_path": "/mnt/HithinkOmniSSD/user_workspace/caisihang/checkpoints/Qwen3.8-27B-FP8",
|
| 6 |
+
"sample_count": 4,
|
| 7 |
+
"records": [
|
| 8 |
+
{
|
| 9 |
+
"sample_id": "0003439",
|
| 10 |
+
"prompt_tokens_in_reference_tensor": 1705,
|
| 11 |
+
"input_shapes": {
|
| 12 |
+
"input_ids": [
|
| 13 |
+
1,
|
| 14 |
+
1705
|
| 15 |
+
],
|
| 16 |
+
"attention_mask": [
|
| 17 |
+
1,
|
| 18 |
+
1705
|
| 19 |
+
],
|
| 20 |
+
"mm_token_type_ids": [
|
| 21 |
+
1,
|
| 22 |
+
1705
|
| 23 |
+
],
|
| 24 |
+
"pixel_values": [
|
| 25 |
+
3888,
|
| 26 |
+
1536
|
| 27 |
+
],
|
| 28 |
+
"image_grid_thw": [
|
| 29 |
+
1,
|
| 30 |
+
3
|
| 31 |
+
]
|
| 32 |
+
},
|
| 33 |
+
"prefill_flops_counted": 90083398713024,
|
| 34 |
+
"decode_one_token_flops_counted": 51922600128,
|
| 35 |
+
"prefill_gflops_counted": 90083.398713024,
|
| 36 |
+
"decode_one_token_gflops_counted": 51.922600128,
|
| 37 |
+
"prefill_operators": {
|
| 38 |
+
"aten.convolution": 20475543552,
|
| 39 |
+
"aten.addmm": 3283626885120,
|
| 40 |
+
"aten.bmm": 3741356981952,
|
| 41 |
+
"aten.mm": 83037939302400
|
| 42 |
+
},
|
| 43 |
+
"decode_operators": {
|
| 44 |
+
"aten.bmm": 670826688,
|
| 45 |
+
"aten.mm": 51243909120,
|
| 46 |
+
"aten.convolution": 7864320
|
| 47 |
+
}
|
| 48 |
+
},
|
| 49 |
+
{
|
| 50 |
+
"sample_id": "0001957",
|
| 51 |
+
"prompt_tokens_in_reference_tensor": 1705,
|
| 52 |
+
"input_shapes": {
|
| 53 |
+
"input_ids": [
|
| 54 |
+
1,
|
| 55 |
+
1705
|
| 56 |
+
],
|
| 57 |
+
"attention_mask": [
|
| 58 |
+
1,
|
| 59 |
+
1705
|
| 60 |
+
],
|
| 61 |
+
"mm_token_type_ids": [
|
| 62 |
+
1,
|
| 63 |
+
1705
|
| 64 |
+
],
|
| 65 |
+
"pixel_values": [
|
| 66 |
+
3888,
|
| 67 |
+
1536
|
| 68 |
+
],
|
| 69 |
+
"image_grid_thw": [
|
| 70 |
+
1,
|
| 71 |
+
3
|
| 72 |
+
]
|
| 73 |
+
},
|
| 74 |
+
"prefill_flops_counted": 90083398713024,
|
| 75 |
+
"decode_one_token_flops_counted": 51922600128,
|
| 76 |
+
"prefill_gflops_counted": 90083.398713024,
|
| 77 |
+
"decode_one_token_gflops_counted": 51.922600128,
|
| 78 |
+
"prefill_operators": {
|
| 79 |
+
"aten.convolution": 20475543552,
|
| 80 |
+
"aten.addmm": 3283626885120,
|
| 81 |
+
"aten.bmm": 3741356981952,
|
| 82 |
+
"aten.mm": 83037939302400
|
| 83 |
+
},
|
| 84 |
+
"decode_operators": {
|
| 85 |
+
"aten.bmm": 670826688,
|
| 86 |
+
"aten.mm": 51243909120,
|
| 87 |
+
"aten.convolution": 7864320
|
| 88 |
+
}
|
| 89 |
+
},
|
| 90 |
+
{
|
| 91 |
+
"sample_id": "0003113",
|
| 92 |
+
"prompt_tokens_in_reference_tensor": 1531,
|
| 93 |
+
"input_shapes": {
|
| 94 |
+
"input_ids": [
|
| 95 |
+
1,
|
| 96 |
+
1531
|
| 97 |
+
],
|
| 98 |
+
"attention_mask": [
|
| 99 |
+
1,
|
| 100 |
+
1531
|
| 101 |
+
],
|
| 102 |
+
"mm_token_type_ids": [
|
| 103 |
+
1,
|
| 104 |
+
1531
|
| 105 |
+
],
|
| 106 |
+
"pixel_values": [
|
| 107 |
+
3192,
|
| 108 |
+
1536
|
| 109 |
+
],
|
| 110 |
+
"image_grid_thw": [
|
| 111 |
+
1,
|
| 112 |
+
3
|
| 113 |
+
]
|
| 114 |
+
},
|
| 115 |
+
"prefill_flops_counted": 80104234744896,
|
| 116 |
+
"decode_one_token_flops_counted": 51854180544,
|
| 117 |
+
"prefill_gflops_counted": 80104.234744896,
|
| 118 |
+
"decode_one_token_gflops_counted": 51.854180544,
|
| 119 |
+
"prefill_operators": {
|
| 120 |
+
"aten.convolution": 17328242688,
|
| 121 |
+
"aten.addmm": 2695817134080,
|
| 122 |
+
"aten.bmm": 2827143609408,
|
| 123 |
+
"aten.mm": 74563945758720
|
| 124 |
+
},
|
| 125 |
+
"decode_operators": {
|
| 126 |
+
"aten.bmm": 602407104,
|
| 127 |
+
"aten.mm": 51243909120,
|
| 128 |
+
"aten.convolution": 7864320
|
| 129 |
+
}
|
| 130 |
+
},
|
| 131 |
+
{
|
| 132 |
+
"sample_id": "0000184",
|
| 133 |
+
"prompt_tokens_in_reference_tensor": 1705,
|
| 134 |
+
"input_shapes": {
|
| 135 |
+
"input_ids": [
|
| 136 |
+
1,
|
| 137 |
+
1705
|
| 138 |
+
],
|
| 139 |
+
"attention_mask": [
|
| 140 |
+
1,
|
| 141 |
+
1705
|
| 142 |
+
],
|
| 143 |
+
"mm_token_type_ids": [
|
| 144 |
+
1,
|
| 145 |
+
1705
|
| 146 |
+
],
|
| 147 |
+
"pixel_values": [
|
| 148 |
+
3888,
|
| 149 |
+
1536
|
| 150 |
+
],
|
| 151 |
+
"image_grid_thw": [
|
| 152 |
+
1,
|
| 153 |
+
3
|
| 154 |
+
]
|
| 155 |
+
},
|
| 156 |
+
"prefill_flops_counted": 90083398713024,
|
| 157 |
+
"decode_one_token_flops_counted": 51922600128,
|
| 158 |
+
"prefill_gflops_counted": 90083.398713024,
|
| 159 |
+
"decode_one_token_gflops_counted": 51.922600128,
|
| 160 |
+
"prefill_operators": {
|
| 161 |
+
"aten.convolution": 20475543552,
|
| 162 |
+
"aten.addmm": 3283626885120,
|
| 163 |
+
"aten.bmm": 3741356981952,
|
| 164 |
+
"aten.mm": 83037939302400
|
| 165 |
+
},
|
| 166 |
+
"decode_operators": {
|
| 167 |
+
"aten.bmm": 670826688,
|
| 168 |
+
"aten.mm": 51243909120,
|
| 169 |
+
"aten.convolution": 7864320
|
| 170 |
+
}
|
| 171 |
+
}
|
| 172 |
+
],
|
| 173 |
+
"mean_prefill_gflops_counted": 87588.607720992,
|
| 174 |
+
"mean_decode_one_token_gflops_counted": 51.905495232,
|
| 175 |
+
"scope": "HF reference execution on the A800 with real task inputs; dense products/convolutions/supported attention counted with multiply-add=2 FLOPs. vLLM custom kernels, elementwise/normalization/nonlinearities and other unsupported operations are not complete.",
|
| 176 |
+
"answer_generation": "not performed; one known-token decode probe only"
|
| 177 |
+
}
|
releases/20260913_lr/paper_data/cost_20260911/router_architecture.json
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"pair_key": "grefcoco|instancevg|MiniCPM",
|
| 4 |
+
"task": "grefcoco",
|
| 5 |
+
"expert": "instancevg",
|
| 6 |
+
"endpoint": "MiniCPM",
|
| 7 |
+
"branches": ["image", "text", "hidden", "confidence", "output"],
|
| 8 |
+
"input_dimensions": {
|
| 9 |
+
"image_flat_plus_valid": 1538,
|
| 10 |
+
"text": 768,
|
| 11 |
+
"hidden": 768,
|
| 12 |
+
"confidence": 8,
|
| 13 |
+
"output": 2
|
| 14 |
+
},
|
| 15 |
+
"representation_dim": 128,
|
| 16 |
+
"head_hidden_dim": 128,
|
| 17 |
+
"class_count": 4,
|
| 18 |
+
"source_checkpoint": "/mnt/HithinkOmniSSD/user_workspace/caisihang/research/ICASSP_ModelCollaboration/outputs/router_method_improvement_20260911_r1/conditions/both_instancevg_MiniCPM_seed42/best.pt"
|
| 19 |
+
},
|
| 20 |
+
{
|
| 21 |
+
"pair_key": "construction|yolo26x|Qwen3.8",
|
| 22 |
+
"task": "construction",
|
| 23 |
+
"expert": "yolo26x",
|
| 24 |
+
"endpoint": "Qwen3.8",
|
| 25 |
+
"branches": ["image", "hidden", "confidence", "output"],
|
| 26 |
+
"input_dimensions": {
|
| 27 |
+
"image_flat_plus_valid": 1538,
|
| 28 |
+
"hidden": 768,
|
| 29 |
+
"confidence": 8,
|
| 30 |
+
"output": 8
|
| 31 |
+
},
|
| 32 |
+
"representation_dim": 128,
|
| 33 |
+
"head_hidden_dim": 128,
|
| 34 |
+
"class_count": 4,
|
| 35 |
+
"source_checkpoint": "/mnt/HithinkOmniSSD/user_workspace/caisihang/research/ICASSP_ModelCollaboration/outputs/router_method_improvement_20260911_r1/conditions/both_yolo26x_Qwen3.8_seed42/best.pt"
|
| 36 |
+
}
|
| 37 |
+
]
|
releases/20260913_lr/paper_data/cost_20260911/yolo26x_specialist.json
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"component": "specialist",
|
| 3 |
+
"expert": "yolo26x",
|
| 4 |
+
"task": "construction",
|
| 5 |
+
"device": "cuda:0",
|
| 6 |
+
"checkpoint": "/mnt/HithinkOmniSSD/user_workspace/caisihang/research/ICASSP_ModelCollaboration/models/construction/yolo26x.pt",
|
| 7 |
+
"sample_count": 64,
|
| 8 |
+
"batch_size": 16,
|
| 9 |
+
"sample_ids": [
|
| 10 |
+
"0000944",
|
| 11 |
+
"0003511",
|
| 12 |
+
"0003365",
|
| 13 |
+
"0001351",
|
| 14 |
+
"0002149",
|
| 15 |
+
"0003368",
|
| 16 |
+
"0000176",
|
| 17 |
+
"0004487",
|
| 18 |
+
"0000013",
|
| 19 |
+
"0003583",
|
| 20 |
+
"0000758",
|
| 21 |
+
"0001291",
|
| 22 |
+
"0001451",
|
| 23 |
+
"0002755",
|
| 24 |
+
"0003088",
|
| 25 |
+
"0004538",
|
| 26 |
+
"0004324",
|
| 27 |
+
"0000516",
|
| 28 |
+
"0003893",
|
| 29 |
+
"0003475",
|
| 30 |
+
"0002137",
|
| 31 |
+
"0003272",
|
| 32 |
+
"0001711",
|
| 33 |
+
"0003262",
|
| 34 |
+
"0001470",
|
| 35 |
+
"0002145",
|
| 36 |
+
"0002662",
|
| 37 |
+
"0000756",
|
| 38 |
+
"0001631",
|
| 39 |
+
"0001924",
|
| 40 |
+
"0002496",
|
| 41 |
+
"0001443",
|
| 42 |
+
"0004053",
|
| 43 |
+
"0002168",
|
| 44 |
+
"0001881",
|
| 45 |
+
"0001666",
|
| 46 |
+
"0000304",
|
| 47 |
+
"0001252",
|
| 48 |
+
"0004844",
|
| 49 |
+
"0000949",
|
| 50 |
+
"0003430",
|
| 51 |
+
"0003807",
|
| 52 |
+
"0004199",
|
| 53 |
+
"0002538",
|
| 54 |
+
"0002091",
|
| 55 |
+
"0001569",
|
| 56 |
+
"0003048",
|
| 57 |
+
"0002854",
|
| 58 |
+
"0000380",
|
| 59 |
+
"0001234",
|
| 60 |
+
"0001484",
|
| 61 |
+
"0003180",
|
| 62 |
+
"0000824",
|
| 63 |
+
"0003784",
|
| 64 |
+
"0000894",
|
| 65 |
+
"0004168",
|
| 66 |
+
"0000564",
|
| 67 |
+
"0004296",
|
| 68 |
+
"0003054",
|
| 69 |
+
"0001583",
|
| 70 |
+
"0003386",
|
| 71 |
+
"0000205",
|
| 72 |
+
"0004214",
|
| 73 |
+
"0003625"
|
| 74 |
+
],
|
| 75 |
+
"model_load_seconds_excluded_from_latency": true,
|
| 76 |
+
"forward_latency_ms_per_sample": [
|
| 77 |
+
28.76109504699707,
|
| 78 |
+
28.29251480102539,
|
| 79 |
+
28.306396484375,
|
| 80 |
+
42.96271896362305
|
| 81 |
+
],
|
| 82 |
+
"end_to_end_latency_ms_per_sample": [
|
| 83 |
+
28.19202906539431,
|
| 84 |
+
27.162021433468908,
|
| 85 |
+
28.371096690534614,
|
| 86 |
+
43.443599628517404
|
| 87 |
+
],
|
| 88 |
+
"mean_forward_latency_ms_per_sample": 32.08068132400513,
|
| 89 |
+
"median_forward_latency_ms_per_sample": 28.533745765686035,
|
| 90 |
+
"mean_end_to_end_latency_ms_per_sample": 31.79218670447881,
|
| 91 |
+
"median_end_to_end_latency_ms_per_sample": 28.28156287796446,
|
| 92 |
+
"flop_profiles": [
|
| 93 |
+
{
|
| 94 |
+
"batch_size": 16,
|
| 95 |
+
"flops_counted": 3110687539200,
|
| 96 |
+
"gflops_counted_per_sample": 194.4179712,
|
| 97 |
+
"operators": {
|
| 98 |
+
"aten.convolution": 3101840179200,
|
| 99 |
+
"aten.bmm": 8847360000
|
| 100 |
+
}
|
| 101 |
+
},
|
| 102 |
+
{
|
| 103 |
+
"batch_size": 16,
|
| 104 |
+
"flops_counted": 3110687539200,
|
| 105 |
+
"gflops_counted_per_sample": 194.4179712,
|
| 106 |
+
"operators": {
|
| 107 |
+
"aten.convolution": 3101840179200,
|
| 108 |
+
"aten.bmm": 8847360000
|
| 109 |
+
}
|
| 110 |
+
},
|
| 111 |
+
{
|
| 112 |
+
"batch_size": 16,
|
| 113 |
+
"flops_counted": 3110687539200,
|
| 114 |
+
"gflops_counted_per_sample": 194.4179712,
|
| 115 |
+
"operators": {
|
| 116 |
+
"aten.convolution": 3101840179200,
|
| 117 |
+
"aten.bmm": 8847360000
|
| 118 |
+
}
|
| 119 |
+
},
|
| 120 |
+
{
|
| 121 |
+
"batch_size": 16,
|
| 122 |
+
"flops_counted": 3110687539200,
|
| 123 |
+
"gflops_counted_per_sample": 194.4179712,
|
| 124 |
+
"operators": {
|
| 125 |
+
"aten.convolution": 3101840179200,
|
| 126 |
+
"aten.bmm": 8847360000
|
| 127 |
+
}
|
| 128 |
+
}
|
| 129 |
+
],
|
| 130 |
+
"mean_counted_gflops_per_sample": 194.4179712,
|
| 131 |
+
"flops_scope": "GPU FlopCounterMode around the real Ultralytics predict path; multiply-add counted as 2 FLOPs. Elementwise, postprocessing and unsupported custom ops may be missing.",
|
| 132 |
+
"latency_scope": "A800 wall-clock latency for real Ultralytics prediction including image loading/preprocess, forward and result postprocessing, excluding model load; batch processing uses the recorded batch size.",
|
| 133 |
+
"completed": true
|
| 134 |
+
}
|
releases/20260913_lr/paper_data/cost_20260911/成本性能补充报告.md
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# A800成本—性能补充报告(2026-09-11)
|
| 2 |
+
|
| 3 |
+
## 1. 做了什么
|
| 4 |
+
|
| 5 |
+
- 在A800上补测了正文中两个代表性模型对:gRefCOCO的InstanceVG + MiniCPM,以及Construction的YOLO26x + Qwen3.8。
|
| 6 |
+
- 分别实测了小模型端、路由器端和大模型端的延迟与可计数运算量;大模型端延迟采用批处理均摊后的真实墙钟时间。
|
| 7 |
+
- 从已经锁定的测试曲线中提取代表性调用预算,按测试集实际调用率换算每个输入的成本,并生成完整曲线。没有用测试集重新挑选模型、阈值或预算。
|
| 8 |
+
|
| 9 |
+
## 2. 为什么做
|
| 10 |
+
|
| 11 |
+
正文原有成本图只是代理估算,缺少InstanceVG、完整解码和真实延迟。此次补测用于回答一个更直接的问题:两项同时改相对置信度路由的性能提升,是否值得它增加的计算和时间成本。
|
| 12 |
+
|
| 13 |
+
## 3. 原始实测成本
|
| 14 |
+
|
| 15 |
+
| 模型对 | 小模型端到端延迟(毫秒/样本) | 小模型可计数运算量(GFLOPs/样本) | 路由器延迟(毫秒/样本) | 路由器稠密运算量(GFLOPs/样本) | 大模型调用延迟(毫秒/样本) | 大模型完整生成估算(GFLOPs/调用) |
|
| 16 |
+
|---|---:|---:|---:|---:|---:|---:|
|
| 17 |
+
| gRefCOCO / InstanceVG + MiniCPM | 16.317 | 7.503 | 0.001831 | 0.000954 | 1447.432 | 10085.742 |
|
| 18 |
+
| Construction / YOLO26x + Qwen3.8 | 31.792 | 194.418 | 0.004676 | 0.000727 | 2599.871 | 94701.391 |
|
| 19 |
+
|
| 20 |
+
说明:GFLOPs按乘加计2次运算,只覆盖计数器能识别的稠密乘法、卷积和注意力等算子;路由器的计数器返回0,因此按已加载检查点的线性层结构精确计算了稠密乘加量。大模型完整生成GFLOPs为“预填充GFLOPs + 平均生成长度 × 单Token解码GFLOPs”的估算,不把未计数算子冒充成完整硬件计数。
|
| 21 |
+
|
| 22 |
+
## 4. 验证集锁定点在测试集上的结果
|
| 23 |
+
|
| 24 |
+
下表的均值和标准差只在三个随机种子之间统计。成本使用对应的平均实际调用率;星号表示验证集选出的运行点。
|
| 25 |
+
|
| 26 |
+
| 模型对 | 方法 | 测试性能 | 实际调用率 | 估算GFLOPs/输入 | 端到端延迟(毫秒/输入) |
|
| 27 |
+
|---|---|---:|---:|---:|---:|
|
| 28 |
+
| gRefCOCO / InstanceVG + MiniCPM | 两项同时改* | 68.05±0.04% | 11.29±1.28% | 1146.415 | 179.767 |
|
| 29 |
+
| gRefCOCO / InstanceVG + MiniCPM | 原方法* | 67.33±0.15% | 26.05±1.96% | 2634.360 | 393.306 |
|
| 30 |
+
| gRefCOCO / InstanceVG + MiniCPM | 置信度路由* | 61.88±0.00% | 52.09±0.00% | 5261.597 | 770.348 |
|
| 31 |
+
| Construction / YOLO26x + Qwen3.8 | 两项同时改* | 30.11±0.14% | 44.14±0.58% | 41996.697 | 1179.410 |
|
| 32 |
+
| Construction / YOLO26x + Qwen3.8 | 原方法* | 30.00±0.22% | 46.67±1.72% | 44392.604 | 1245.185 |
|
| 33 |
+
| Construction / YOLO26x + Qwen3.8 | 置信度路由* | 25.97±0.00% | 92.21±0.00% | 87518.937 | 2429.148 |
|
| 34 |
+
|
| 35 |
+
### 两项同时改相对置信度路由
|
| 36 |
+
|
| 37 |
+
| 模型对 | 性能变化 | 调用率变化 | 估算GFLOPs变化 | 实际延迟变化 |
|
| 38 |
+
|---|---:|---:|---:|---:|
|
| 39 |
+
| gRefCOCO / InstanceVG + MiniCPM | +6.18 个百分点 | -40.80 个百分点 | -78.2% | -76.7% |
|
| 40 |
+
| Construction / YOLO26x + Qwen3.8 | +4.13 个百分点 | -48.07 个百分点 | -52.0% | -51.4% |
|
| 41 |
+
|
| 42 |
+
直白地说:两项同时改在这两个代表性模型对上都提高了测试性能,同时减少了实际调用,因此计算量和延迟也一起下降。原因不是路由器本身更快,而是它让大模型少被调用;路由器自身成本相对很小。
|
| 43 |
+
|
| 44 |
+
## 5. 固定名义调用预算的代表点
|
| 45 |
+
|
| 46 |
+
名义预算是验证阶段设定的阈值预算;阈值原样转到测试集后,实际调用率会有偏差。完整的99个阈值点和端点均在CSV中。
|
| 47 |
+
|
| 48 |
+
| 模型对 | 名义预算 | 两项同时改:性能 / 实际调用 | 置信度:性能 / 实际调用 | 两项同时改:GFLOPs/输入 | 两项同时改:毫秒/输入 |
|
| 49 |
+
|---|---:|---:|---:|---:|---:|
|
| 50 |
+
| gRefCOCO / InstanceVG + MiniCPM | 25% | 67.94% / 24.53% | 65.97% / 12.48% | 2481.628 | 371.387 |
|
| 51 |
+
| gRefCOCO / InstanceVG + MiniCPM | 50% | 66.46% / 43.10% | 65.28% / 23.45% | 4354.834 | 640.216 |
|
| 52 |
+
| Construction / YOLO26x + Qwen3.8 | 25% | 25.62% / 26.50% | 18.03% / 24.80% | 25288.396 | 720.711 |
|
| 53 |
+
| Construction / YOLO26x + Qwen3.8 | 50% | 29.98% / 49.51% | 18.02% / 50.13% | 47082.746 | 1319.039 |
|
| 54 |
+
|
| 55 |
+
## 6. 结果怎么理解
|
| 56 |
+
|
| 57 |
+
1. gRefCOCO的InstanceVG + MiniCPM是这两组中相对置信度提升更明显的一组;Construction的YOLO26x + Qwen3.8也有稳定的正向变化。
|
| 58 |
+
2. 这里的成本是按真实测试调用率换算的平均每输入成本,不是把名义预算当成实际调用率。
|
| 59 |
+
3. 大模型延迟是A800、批量4下的均摊值,模型加载和预热没有计入;实际系统的并发度、批量大小和请求排队会改变墙钟延迟。
|
| 60 |
+
4. InstanceVG使用了远端已有的MMCV等价算子兼容路径,因为原生detrex扩展不可用;这不改变本次模型输出链路,但应在复现实验时保留该环境说明。
|
| 61 |
+
5. 本补充不修改正文,也不覆盖旧实验结果;它是针对两个代表性模型对的成本实测和成本—性能换算。
|
| 62 |
+
|
| 63 |
+
## 7. 文件在哪里
|
| 64 |
+
|
| 65 |
+
- 原始A800测量记录:`C:/Users/csh10/Documents/CodexProjects/ICASSP-COLLA/dse_results/method_improvement_20260911/cost_analysis_20260911`目录下的8个JSON文件。
|
| 66 |
+
- 完整成本—性能曲线数据:`C:/Users/csh10/Documents/CodexProjects/ICASSP-COLLA/dse_results/method_improvement_20260911/cost_analysis_20260911/full_cost_performance_curves.csv`。
|
| 67 |
+
- 固定预算代表点:`C:/Users/csh10/Documents/CodexProjects/ICASSP-COLLA/dse_results/method_improvement_20260911/cost_analysis_20260911/fixed_budget_cost_performance.csv`。
|
| 68 |
+
- FLOPs—性能图:`C:/Users/csh10/Documents/CodexProjects/ICASSP-COLLA/dse_results/method_improvement_20260911/cost_analysis_20260911/flops_performance.pdf`;实际延迟—性能图:`C:/Users/csh10/Documents/CodexProjects/ICASSP-COLLA/dse_results/method_improvement_20260911/cost_analysis_20260911/latency_performance.pdf`。
|
| 69 |
+
- 可复现脚本:`C:/Users/csh10/Documents/CodexProjects/ICASSP-COLLA/dse_results/method_improvement_20260911/cost_analysis_20260911/make_cost_analysis_20260911.py`;插入论文的LaTeX片段:`C:/Users/csh10/Documents/CodexProjects/ICASSP-COLLA/dse_results/method_improvement_20260911/cost_analysis_20260911/latex_includes.tex`。
|
releases/20260913_lr/paper_data/cost_components.json
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"components": {
|
| 3 |
+
"grefcoco|instancevg|MiniCPM": {
|
| 4 |
+
"specialist_gflops": 7.502680064,
|
| 5 |
+
"router_gflops": 3.642171392,
|
| 6 |
+
"large_gflops": 12916.147709832,
|
| 7 |
+
"specialist_ms": 46.08042127620138,
|
| 8 |
+
"router_ms": 13.180577338061994,
|
| 9 |
+
"large_ms": 3667.173011115665,
|
| 10 |
+
"large_samples": 64,
|
| 11 |
+
"flops_probes": 8,
|
| 12 |
+
"specialist_samples": 64,
|
| 13 |
+
"router_samples": 64,
|
| 14 |
+
"batch_size": 1
|
| 15 |
+
},
|
| 16 |
+
"construction|yolo26x|Qwen3.8": {
|
| 17 |
+
"specialist_gflops": 135.97853184000002,
|
| 18 |
+
"router_gflops": 3.183754752,
|
| 19 |
+
"large_gflops": 96874.712551641,
|
| 20 |
+
"specialist_ms": 46.30809425361804,
|
| 21 |
+
"router_ms": 25.261267359383055,
|
| 22 |
+
"large_ms": 9822.475872695577,
|
| 23 |
+
"large_samples": 64,
|
| 24 |
+
"flops_probes": 8,
|
| 25 |
+
"specialist_samples": 64,
|
| 26 |
+
"router_samples": 64,
|
| 27 |
+
"batch_size": 1
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"protocol": {
|
| 31 |
+
"method": "original main-table logit contrast; original checkpoints and validation thresholds; seed 42",
|
| 32 |
+
"large_timing": "64 measured examples; batch size one; same full task prompt; two separate warmup examples",
|
| 33 |
+
"cost": "serial component estimates, not a timed integrated service; includes active distilled encoders and routing head only for learned",
|
| 34 |
+
"flops": "multiply-add=2; supported operators; mean prefill (including visual embedding) plus average generated length times one-token decode probe",
|
| 35 |
+
"scope_limits": "mean call cost extrapolated across policies; no policy-specific input/output-length costs; specialist-feature extraction overhead is not separately measured; confidence threshold overhead not separately timed"
|
| 36 |
+
},
|
| 37 |
+
"source_files": [
|
| 38 |
+
"dse_results/main_table_cost_20260912/cost_summary.json",
|
| 39 |
+
"dse_results/main_table_cost_qwen_20260912/cost_summary.json"
|
| 40 |
+
]
|
| 41 |
+
}
|
releases/20260913_lr/paper_data/cost_five_seeds.json
ADDED
|
@@ -0,0 +1,3299 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"components": {
|
| 3 |
+
"grefcoco|instancevg|MiniCPM": {
|
| 4 |
+
"specialist_gflops": 7.502680064,
|
| 5 |
+
"router_gflops": 3.642171392,
|
| 6 |
+
"large_gflops": 12916.147709832,
|
| 7 |
+
"specialist_ms": 46.08042127620138,
|
| 8 |
+
"router_ms": 13.180577338061994,
|
| 9 |
+
"large_ms": 3667.173011115665,
|
| 10 |
+
"large_samples": 64,
|
| 11 |
+
"flops_probes": 8,
|
| 12 |
+
"specialist_samples": 64,
|
| 13 |
+
"router_samples": 64,
|
| 14 |
+
"batch_size": 1
|
| 15 |
+
},
|
| 16 |
+
"construction|yolo26x|Qwen3.8": {
|
| 17 |
+
"specialist_gflops": 135.97853184000002,
|
| 18 |
+
"router_gflops": 3.183754752,
|
| 19 |
+
"large_gflops": 96874.712551641,
|
| 20 |
+
"specialist_ms": 46.30809425361804,
|
| 21 |
+
"router_ms": 25.261267359383055,
|
| 22 |
+
"large_ms": 9822.475872695577,
|
| 23 |
+
"large_samples": 64,
|
| 24 |
+
"flops_probes": 8,
|
| 25 |
+
"specialist_samples": 64,
|
| 26 |
+
"router_samples": 64,
|
| 27 |
+
"batch_size": 1
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"selected": [
|
| 31 |
+
{
|
| 32 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 33 |
+
"method": "learned",
|
| 34 |
+
"metric": 0.6791367722542041,
|
| 35 |
+
"calls": 0.19038085245157815,
|
| 36 |
+
"gflops": 2470.132062844315,
|
| 37 |
+
"ms": 757.4205225578843
|
| 38 |
+
},
|
| 39 |
+
{
|
| 40 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 41 |
+
"method": "confidence",
|
| 42 |
+
"metric": 0.6187788900547316,
|
| 43 |
+
"calls": 0.5209426310864078,
|
| 44 |
+
"gflops": 6736.074651524562,
|
| 45 |
+
"ms": 1956.4671783358606
|
| 46 |
+
},
|
| 47 |
+
{
|
| 48 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 49 |
+
"method": "learned",
|
| 50 |
+
"metric": 0.2979678330127048,
|
| 51 |
+
"calls": 0.44766977363515315,
|
| 52 |
+
"gflops": 43507.04292555566,
|
| 53 |
+
"ms": 4468.794912079384
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 57 |
+
"method": "confidence",
|
| 58 |
+
"metric": 0.2597490004762105,
|
| 59 |
+
"calls": 0.9221038615179761,
|
| 60 |
+
"gflops": 89464.5250591521,
|
| 61 |
+
"ms": 9103.651026133362
|
| 62 |
+
}
|
| 63 |
+
],
|
| 64 |
+
"curves": [
|
| 65 |
+
{
|
| 66 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 67 |
+
"method": "learned",
|
| 68 |
+
"metric": 0.6695119530385957,
|
| 69 |
+
"calls": 0.0,
|
| 70 |
+
"gflops": 11.144851456,
|
| 71 |
+
"ms": 59.26099861426337
|
| 72 |
+
},
|
| 73 |
+
{
|
| 74 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 75 |
+
"method": "learned",
|
| 76 |
+
"metric": 0.6736239117488585,
|
| 77 |
+
"calls": 0.020332926863851627,
|
| 78 |
+
"gflops": 273.76793820271877,
|
| 79 |
+
"ms": 133.82535924636875
|
| 80 |
+
},
|
| 81 |
+
{
|
| 82 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 83 |
+
"method": "learned",
|
| 84 |
+
"metric": 0.6766298953577403,
|
| 85 |
+
"calls": 0.04462467742392876,
|
| 86 |
+
"gflops": 587.5237765670691,
|
| 87 |
+
"ms": 222.90741129303743
|
| 88 |
+
},
|
| 89 |
+
{
|
| 90 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 91 |
+
"method": "learned",
|
| 92 |
+
"metric": 0.6786603522105323,
|
| 93 |
+
"calls": 0.07091285483367836,
|
| 94 |
+
"gflops": 927.0657590136638,
|
| 95 |
+
"ms": 319.31070600149167
|
| 96 |
+
},
|
| 97 |
+
{
|
| 98 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 99 |
+
"method": "learned",
|
| 100 |
+
"metric": 0.6797663273119132,
|
| 101 |
+
"calls": 0.09045174829141027,
|
| 102 |
+
"gflops": 1179.4329930003992,
|
| 103 |
+
"ms": 390.9632087567506
|
| 104 |
+
},
|
| 105 |
+
{
|
| 106 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 107 |
+
"method": "learned",
|
| 108 |
+
"metric": 0.6804072257039957,
|
| 109 |
+
"calls": 0.11004735842100785,
|
| 110 |
+
"gflops": 1432.5327878985618,
|
| 111 |
+
"ms": 462.8237013603555
|
| 112 |
+
},
|
| 113 |
+
{
|
| 114 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 115 |
+
"method": "learned",
|
| 116 |
+
"metric": 0.6805717040524062,
|
| 117 |
+
"calls": 0.12548563650285,
|
| 118 |
+
"gflops": 1631.9358679890966,
|
| 119 |
+
"ms": 519.4385380801856
|
| 120 |
+
},
|
| 121 |
+
{
|
| 122 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 123 |
+
"method": "learned",
|
| 124 |
+
"metric": 0.6804015540368091,
|
| 125 |
+
"calls": 0.13746986926807134,
|
| 126 |
+
"gflops": 1786.725988573704,
|
| 127 |
+
"ms": 563.3867930357333
|
| 128 |
+
},
|
| 129 |
+
{
|
| 130 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 131 |
+
"method": "learned",
|
| 132 |
+
"metric": 0.6803391656977569,
|
| 133 |
+
"calls": 0.15023112043785272,
|
| 134 |
+
"gflops": 1951.5521936448667,
|
| 135 |
+
"ms": 610.1845089136239
|
| 136 |
+
},
|
| 137 |
+
{
|
| 138 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 139 |
+
"method": "learned",
|
| 140 |
+
"metric": 0.6802767773587046,
|
| 141 |
+
"calls": 0.16403595836996285,
|
| 142 |
+
"gflops": 2129.857519486293,
|
| 143 |
+
"ms": 660.8092380010838
|
| 144 |
+
},
|
| 145 |
+
{
|
| 146 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 147 |
+
"method": "learned",
|
| 148 |
+
"metric": 0.6802881206930778,
|
| 149 |
+
"calls": 0.1779372146442447,
|
| 150 |
+
"gflops": 2309.4081988771463,
|
| 151 |
+
"ms": 711.7875498307326
|
| 152 |
+
},
|
| 153 |
+
{
|
| 154 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 155 |
+
"method": "learned",
|
| 156 |
+
"metric": 0.6802824490258912,
|
| 157 |
+
"calls": 0.18960383404701814,
|
| 158 |
+
"gflops": 2460.09597835776,
|
| 159 |
+
"ms": 754.5710616355417
|
| 160 |
+
},
|
| 161 |
+
{
|
| 162 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 163 |
+
"method": "learned",
|
| 164 |
+
"metric": 0.6801293140118538,
|
| 165 |
+
"calls": 0.2002041800187165,
|
| 166 |
+
"gflops": 2597.011612703539,
|
| 167 |
+
"ms": 793.4443642914426
|
| 168 |
+
},
|
| 169 |
+
{
|
| 170 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 171 |
+
"method": "learned",
|
| 172 |
+
"metric": 0.6800839406743613,
|
| 173 |
+
"calls": 0.2103564642826759,
|
| 174 |
+
"gflops": 2728.1400158490414,
|
| 175 |
+
"ms": 830.6745471454087
|
| 176 |
+
},
|
| 177 |
+
{
|
| 178 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 179 |
+
"method": "learned",
|
| 180 |
+
"metric": 0.6799818506650028,
|
| 181 |
+
"calls": 0.21922695176247053,
|
| 182 |
+
"gflops": 2842.7125423962843,
|
| 183 |
+
"ms": 863.2041594267511
|
| 184 |
+
},
|
| 185 |
+
{
|
| 186 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 187 |
+
"method": "learned",
|
| 188 |
+
"metric": 0.6798684173212717,
|
| 189 |
+
"calls": 0.2273317641720784,
|
| 190 |
+
"gflops": 2947.395496639259,
|
| 191 |
+
"ms": 892.9259087554203
|
| 192 |
+
},
|
| 193 |
+
{
|
| 194 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 195 |
+
"method": "learned",
|
| 196 |
+
"metric": 0.6796925956384878,
|
| 197 |
+
"calls": 0.2348580665286561,
|
| 198 |
+
"gflops": 3044.606329585673,
|
| 199 |
+
"ms": 920.5261616309583
|
| 200 |
+
},
|
| 201 |
+
{
|
| 202 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 203 |
+
"method": "learned",
|
| 204 |
+
"metric": 0.6796018489635027,
|
| 205 |
+
"calls": 0.24202705385247994,
|
| 206 |
+
"gflops": 3137.202028790095,
|
| 207 |
+
"ms": 946.8160784619154
|
| 208 |
+
},
|
| 209 |
+
{
|
| 210 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 211 |
+
"method": "learned",
|
| 212 |
+
"metric": 0.6794316989479057,
|
| 213 |
+
"calls": 0.24939454952783371,
|
| 214 |
+
"gflops": 3232.361691184513,
|
| 215 |
+
"ms": 973.8339597620842
|
| 216 |
+
},
|
| 217 |
+
{
|
| 218 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 219 |
+
"method": "learned",
|
| 220 |
+
"metric": 0.6793693106088534,
|
| 221 |
+
"calls": 0.25641040183762015,
|
| 222 |
+
"gflops": 3322.9794759280803,
|
| 223 |
+
"ms": 999.5623040025065
|
| 224 |
+
},
|
| 225 |
+
{
|
| 226 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 227 |
+
"method": "learned",
|
| 228 |
+
"metric": 0.6792615489323086,
|
| 229 |
+
"calls": 0.2633184924708618,
|
| 230 |
+
"gflops": 3412.2053949399365,
|
| 231 |
+
"ms": 1024.895467531071
|
| 232 |
+
},
|
| 233 |
+
{
|
| 234 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 235 |
+
"method": "learned",
|
| 236 |
+
"metric": 0.6791027422510847,
|
| 237 |
+
"calls": 0.2692510563480135,
|
| 238 |
+
"gflops": 3488.8312662752414,
|
| 239 |
+
"ms": 1046.6512056680817
|
| 240 |
+
},
|
| 241 |
+
{
|
| 242 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 243 |
+
"method": "learned",
|
| 244 |
+
"metric": 0.6789496072370473,
|
| 245 |
+
"calls": 0.27534242690638916,
|
| 246 |
+
"gflops": 3567.508308162543,
|
| 247 |
+
"ms": 1068.9893153804614
|
| 248 |
+
},
|
| 249 |
+
{
|
| 250 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 251 |
+
"method": "learned",
|
| 252 |
+
"metric": 0.6787681138870771,
|
| 253 |
+
"calls": 0.2815245441397499,
|
| 254 |
+
"gflops": 3647.3574475081286,
|
| 255 |
+
"ms": 1091.660208850195
|
| 256 |
+
},
|
| 257 |
+
{
|
| 258 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 259 |
+
"method": "learned",
|
| 260 |
+
"metric": 0.6786490088761592,
|
| 261 |
+
"calls": 0.28715650965601336,
|
| 262 |
+
"gflops": 3720.1007460128676,
|
| 263 |
+
"ms": 1112.3136007909704
|
| 264 |
+
},
|
| 265 |
+
{
|
| 266 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 267 |
+
"method": "learned",
|
| 268 |
+
"metric": 0.6784902021949353,
|
| 269 |
+
"calls": 0.29289056518163514,
|
| 270 |
+
"gflops": 3794.162654158177,
|
| 271 |
+
"ms": 1133.3413744587692
|
| 272 |
+
},
|
| 273 |
+
{
|
| 274 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 275 |
+
"method": "learned",
|
| 276 |
+
"metric": 0.6783540821824576,
|
| 277 |
+
"calls": 0.2986983523806823,
|
| 278 |
+
"gflops": 3869.1768914883414,
|
| 279 |
+
"ms": 1154.639534929418
|
| 280 |
+
},
|
| 281 |
+
{
|
| 282 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 283 |
+
"method": "learned",
|
| 284 |
+
"metric": 0.6781215438278082,
|
| 285 |
+
"calls": 0.30438703456881144,
|
| 286 |
+
"gflops": 3942.6527509045077,
|
| 287 |
+
"ms": 1175.5009167185397
|
| 288 |
+
},
|
| 289 |
+
{
|
| 290 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 291 |
+
"method": "learned",
|
| 292 |
+
"metric": 0.6777301987919349,
|
| 293 |
+
"calls": 0.31002467175226156,
|
| 294 |
+
"gflops": 4015.469305500391,
|
| 295 |
+
"ms": 1196.17510764415
|
| 296 |
+
},
|
| 297 |
+
{
|
| 298 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 299 |
+
"method": "learned",
|
| 300 |
+
"metric": 0.6775146754388452,
|
| 301 |
+
"calls": 0.31561126393103256,
|
| 302 |
+
"gflops": 4087.626555275989,
|
| 303 |
+
"ms": 1216.662107706249
|
| 304 |
+
},
|
| 305 |
+
{
|
| 306 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 307 |
+
"method": "learned",
|
| 308 |
+
"metric": 0.6773048237529421,
|
| 309 |
+
"calls": 0.3206477043927063,
|
| 310 |
+
"gflops": 4152.677964210741,
|
| 311 |
+
"ms": 1235.1316062393896
|
| 312 |
+
},
|
| 313 |
+
{
|
| 314 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 315 |
+
"method": "learned",
|
| 316 |
+
"metric": 0.6772084054107704,
|
| 317 |
+
"calls": 0.3267560899526416,
|
| 318 |
+
"gflops": 4231.57477437147,
|
| 319 |
+
"ms": 1257.5321129062731
|
| 320 |
+
},
|
| 321 |
+
{
|
| 322 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 323 |
+
"method": "learned",
|
| 324 |
+
"metric": 0.6769191503842554,
|
| 325 |
+
"calls": 0.3326829821626067,
|
| 326 |
+
"gflops": 4308.127389615633,
|
| 327 |
+
"ms": 1279.2670520584488
|
| 328 |
+
},
|
| 329 |
+
{
|
| 330 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 331 |
+
"method": "learned",
|
| 332 |
+
"metric": 0.6765788503530613,
|
| 333 |
+
"calls": 0.33824688767263134,
|
| 334 |
+
"gflops": 4379.991615026658,
|
| 335 |
+
"ms": 1299.6708561812088
|
| 336 |
+
},
|
| 337 |
+
{
|
| 338 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 339 |
+
"method": "learned",
|
| 340 |
+
"metric": 0.6762782519921732,
|
| 341 |
+
"calls": 0.3451209483027536,
|
| 342 |
+
"gflops": 4468.777997491658,
|
| 343 |
+
"ms": 1324.8792258007659
|
| 344 |
+
},
|
| 345 |
+
{
|
| 346 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 347 |
+
"method": "learned",
|
| 348 |
+
"metric": 0.6760173553015909,
|
| 349 |
+
"calls": 0.3511272438533307,
|
| 350 |
+
"gflops": 4546.356198011819,
|
| 351 |
+
"ms": 1346.9053507406265
|
| 352 |
+
},
|
| 353 |
+
{
|
| 354 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 355 |
+
"method": "learned",
|
| 356 |
+
"metric": 0.6754955619204265,
|
| 357 |
+
"calls": 0.35775175112724383,
|
| 358 |
+
"gflops": 4631.919312466537,
|
| 359 |
+
"ms": 1371.19856502746
|
| 360 |
+
},
|
| 361 |
+
{
|
| 362 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 363 |
+
"method": "learned",
|
| 364 |
+
"metric": 0.6750021268751949,
|
| 365 |
+
"calls": 0.36507387346510506,
|
| 366 |
+
"gflops": 4726.492926131814,
|
| 367 |
+
"ms": 1398.0500544489519
|
| 368 |
+
},
|
| 369 |
+
{
|
| 370 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 371 |
+
"method": "learned",
|
| 372 |
+
"metric": 0.6744406318237245,
|
| 373 |
+
"calls": 0.3725207724810708,
|
| 374 |
+
"gflops": 4822.678173802229,
|
| 375 |
+
"ms": 1425.3591215368053
|
| 376 |
+
},
|
| 377 |
+
{
|
| 378 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 379 |
+
"method": "learned",
|
| 380 |
+
"metric": 0.6740152567847318,
|
| 381 |
+
"calls": 0.37905453307999887,
|
| 382 |
+
"gflops": 4907.069190798665,
|
| 383 |
+
"ms": 1449.3195520662853
|
| 384 |
+
},
|
| 385 |
+
{
|
| 386 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 387 |
+
"method": "learned",
|
| 388 |
+
"metric": 0.6733346567223435,
|
| 389 |
+
"calls": 0.38662053710688254,
|
| 390 |
+
"gflops": 5004.792816383078,
|
| 391 |
+
"ms": 1477.0653978356654
|
| 392 |
+
},
|
| 393 |
+
{
|
| 394 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 395 |
+
"method": "learned",
|
| 396 |
+
"metric": 0.6728752516802314,
|
| 397 |
+
"calls": 0.39437937781810967,
|
| 398 |
+
"gflops": 5105.007149066346,
|
| 399 |
+
"ms": 1505.5184090894231
|
| 400 |
+
},
|
| 401 |
+
{
|
| 402 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 403 |
+
"method": "learned",
|
| 404 |
+
"metric": 0.6721946516178432,
|
| 405 |
+
"calls": 0.40177523182939623,
|
| 406 |
+
"gflops": 5200.533091916476,
|
| 407 |
+
"ms": 1532.6402853137647
|
| 408 |
+
},
|
| 409 |
+
{
|
| 410 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 411 |
+
"method": "learned",
|
| 412 |
+
"metric": 0.6716388282335591,
|
| 413 |
+
"calls": 0.4089045174829141,
|
| 414 |
+
"gflops": 5292.6159984829,
|
| 415 |
+
"ms": 1558.7846092508796
|
| 416 |
+
},
|
| 417 |
+
{
|
| 418 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 419 |
+
"method": "learned",
|
| 420 |
+
"metric": 0.6709298698352381,
|
| 421 |
+
"calls": 0.4166690298613277,
|
| 422 |
+
"gflops": 5392.903587257309,
|
| 423 |
+
"ms": 1587.2584194894714
|
| 424 |
+
},
|
| 425 |
+
{
|
| 426 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 427 |
+
"method": "learned",
|
| 428 |
+
"metric": 0.670351359782208,
|
| 429 |
+
"calls": 0.42493832061934605,
|
| 430 |
+
"gflops": 5499.710968143422,
|
| 431 |
+
"ms": 1617.5833393783444
|
| 432 |
+
},
|
| 433 |
+
{
|
| 434 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 435 |
+
"method": "learned",
|
| 436 |
+
"metric": 0.6695743413776479,
|
| 437 |
+
"calls": 0.43286731134617024,
|
| 438 |
+
"gflops": 5602.122983560971,
|
| 439 |
+
"ms": 1646.6603201771404
|
| 440 |
+
},
|
| 441 |
+
{
|
| 442 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 443 |
+
"method": "learned",
|
| 444 |
+
"metric": 0.6687065762981028,
|
| 445 |
+
"calls": 0.4409891387573377,
|
| 446 |
+
"gflops": 5707.025706077373,
|
| 447 |
+
"ms": 1676.4444664603132
|
| 448 |
+
},
|
| 449 |
+
{
|
| 450 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 451 |
+
"method": "learned",
|
| 452 |
+
"metric": 0.6676913478717068,
|
| 453 |
+
"calls": 0.4495646995434308,
|
| 454 |
+
"gflops": 5817.788915885195,
|
| 455 |
+
"ms": 1707.8925315302556
|
| 456 |
+
},
|
| 457 |
+
{
|
| 458 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 459 |
+
"method": "learned",
|
| 460 |
+
"metric": 0.6667838811218558,
|
| 461 |
+
"calls": 0.45795309531236705,
|
| 462 |
+
"gflops": 5926.134674685305,
|
| 463 |
+
"ms": 1738.6542301006555
|
| 464 |
+
},
|
| 465 |
+
{
|
| 466 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 467 |
+
"method": "learned",
|
| 468 |
+
"metric": 0.665910444375124,
|
| 469 |
+
"calls": 0.46621671440319884,
|
| 470 |
+
"gflops": 6032.868799480276,
|
| 471 |
+
"ms": 1768.9583510046941
|
| 472 |
+
},
|
| 473 |
+
{
|
| 474 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 475 |
+
"method": "learned",
|
| 476 |
+
"metric": 0.6649065592831013,
|
| 477 |
+
"calls": 0.47530839690326976,
|
| 478 |
+
"gflops": 6150.298313582086,
|
| 479 |
+
"ms": 1802.2991236945866
|
| 480 |
+
},
|
| 481 |
+
{
|
| 482 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 483 |
+
"method": "learned",
|
| 484 |
+
"metric": 0.6638005841817202,
|
| 485 |
+
"calls": 0.4846496327595496,
|
| 486 |
+
"gflops": 6270.951095694176,
|
| 487 |
+
"ms": 1836.555051717202
|
| 488 |
+
},
|
| 489 |
+
{
|
| 490 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 491 |
+
"method": "learned",
|
| 492 |
+
"metric": 0.6626946090803392,
|
| 493 |
+
"calls": 0.4937072852565011,
|
| 494 |
+
"gflops": 6387.94107324913,
|
| 495 |
+
"ms": 1869.771030498087
|
| 496 |
+
},
|
| 497 |
+
{
|
| 498 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 499 |
+
"method": "learned",
|
| 500 |
+
"metric": 0.6615659473102118,
|
| 501 |
+
"calls": 0.5040467345376174,
|
| 502 |
+
"gflops": 6521.486927402344,
|
| 503 |
+
"ms": 1907.687579851596
|
| 504 |
+
},
|
| 505 |
+
{
|
| 506 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 507 |
+
"method": "learned",
|
| 508 |
+
"metric": 0.6605280322150696,
|
| 509 |
+
"calls": 0.5130249836939569,
|
| 510 |
+
"gflops": 6637.4513196812995,
|
| 511 |
+
"ms": 1940.612372844796
|
| 512 |
+
},
|
| 513 |
+
{
|
| 514 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 515 |
+
"method": "learned",
|
| 516 |
+
"metric": 0.6591214587528004,
|
| 517 |
+
"calls": 0.5229674162720132,
|
| 518 |
+
"gflops": 6765.8692474545205,
|
| 519 |
+
"ms": 1977.0729932598813
|
| 520 |
+
},
|
| 521 |
+
{
|
| 522 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 523 |
+
"method": "learned",
|
| 524 |
+
"metric": 0.6579020503076879,
|
| 525 |
+
"calls": 0.5324504438079574,
|
| 526 |
+
"gflops": 6888.35343184518,
|
| 527 |
+
"ms": 2011.8488959033625
|
| 528 |
+
},
|
| 529 |
+
{
|
| 530 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 531 |
+
"method": "learned",
|
| 532 |
+
"metric": 0.6562969684938889,
|
| 533 |
+
"calls": 0.5418654113376628,
|
| 534 |
+
"gflops": 7009.9585431421265,
|
| 535 |
+
"ms": 2046.3752107288285
|
| 536 |
+
},
|
| 537 |
+
{
|
| 538 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 539 |
+
"method": "learned",
|
| 540 |
+
"metric": 0.6548733800300598,
|
| 541 |
+
"calls": 0.5520120239344355,
|
| 542 |
+
"gflops": 7141.013690196485,
|
| 543 |
+
"ms": 2083.58459459796
|
| 544 |
+
},
|
| 545 |
+
{
|
| 546 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 547 |
+
"method": "learned",
|
| 548 |
+
"metric": 0.6530811331991039,
|
| 549 |
+
"calls": 0.5628449082607834,
|
| 550 |
+
"gflops": 7280.932824279119,
|
| 551 |
+
"ms": 2123.3106556320804
|
| 552 |
+
},
|
| 553 |
+
{
|
| 554 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 555 |
+
"method": "learned",
|
| 556 |
+
"metric": 0.6512435130306553,
|
| 557 |
+
"calls": 0.5729631625216232,
|
| 558 |
+
"gflops": 7411.621690877762,
|
| 559 |
+
"ms": 2160.416044577038
|
| 560 |
+
},
|
| 561 |
+
{
|
| 562 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 563 |
+
"method": "learned",
|
| 564 |
+
"metric": 0.6493945495278337,
|
| 565 |
+
"calls": 0.5823667867169554,
|
| 566 |
+
"gflops": 7533.080289992424,
|
| 567 |
+
"ms": 2194.9007614328348
|
| 568 |
+
},
|
| 569 |
+
{
|
| 570 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 571 |
+
"method": "learned",
|
| 572 |
+
"metric": 0.647755437710915,
|
| 573 |
+
"calls": 0.5923942943028104,
|
| 574 |
+
"gflops": 7662.597059132788,
|
| 575 |
+
"ms": 2231.67336662044
|
| 576 |
+
},
|
| 577 |
+
{
|
| 578 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 579 |
+
"method": "learned",
|
| 580 |
+
"metric": 0.6456966225221904,
|
| 581 |
+
"calls": 0.6029662819385759,
|
| 582 |
+
"gflops": 7799.146413022853,
|
| 583 |
+
"ms": 2270.4426743521676
|
| 584 |
+
},
|
| 585 |
+
{
|
| 586 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 587 |
+
"method": "learned",
|
| 588 |
+
"metric": 0.6438760173553015,
|
| 589 |
+
"calls": 0.6129314011853785,
|
| 590 |
+
"gflops": 7927.857365160644,
|
| 591 |
+
"ms": 2306.986490706591
|
| 592 |
+
},
|
| 593 |
+
{
|
| 594 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 595 |
+
"method": "learned",
|
| 596 |
+
"metric": 0.641459887133823,
|
| 597 |
+
"calls": 0.6240138388679352,
|
| 598 |
+
"gflops": 8070.999767253555,
|
| 599 |
+
"ms": 2347.6277070734345
|
| 600 |
+
},
|
| 601 |
+
{
|
| 602 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 603 |
+
"method": "learned",
|
| 604 |
+
"metric": 0.6394010719450983,
|
| 605 |
+
"calls": 0.6343873181521709,
|
| 606 |
+
"gflops": 8204.985157953626,
|
| 607 |
+
"ms": 2385.669050335951
|
| 608 |
+
},
|
| 609 |
+
{
|
| 610 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 611 |
+
"method": "learned",
|
| 612 |
+
"metric": 0.6372458384142019,
|
| 613 |
+
"calls": 0.6447324391004736,
|
| 614 |
+
"gflops": 8338.604268197982,
|
| 615 |
+
"ms": 2423.6063986742943
|
| 616 |
+
},
|
| 617 |
+
{
|
| 618 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 619 |
+
"method": "learned",
|
| 620 |
+
"metric": 0.6348240365255367,
|
| 621 |
+
"calls": 0.655871593454896,
|
| 622 |
+
"gflops": 8482.479231202318,
|
| 623 |
+
"ms": 2464.4556048894833
|
| 624 |
+
},
|
| 625 |
+
{
|
| 626 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 627 |
+
"method": "learned",
|
| 628 |
+
"metric": 0.6324419363071774,
|
| 629 |
+
"calls": 0.6665116410969005,
|
| 630 |
+
"gflops": 8619.907658186099,
|
| 631 |
+
"ms": 2503.4745004392275
|
| 632 |
+
},
|
| 633 |
+
{
|
| 634 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 635 |
+
"method": "learned",
|
| 636 |
+
"metric": 0.6297138643904374,
|
| 637 |
+
"calls": 0.6780421404871962,
|
| 638 |
+
"gflops": 8768.837291479285,
|
| 639 |
+
"ms": 2545.7588366080054
|
| 640 |
+
},
|
| 641 |
+
{
|
| 642 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 643 |
+
"method": "learned",
|
| 644 |
+
"metric": 0.6270595241471231,
|
| 645 |
+
"calls": 0.687848453052775,
|
| 646 |
+
"gflops": 8895.497073065082,
|
| 647 |
+
"ms": 2581.7202813870604
|
| 648 |
+
},
|
| 649 |
+
{
|
| 650 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 651 |
+
"method": "learned",
|
| 652 |
+
"metric": 0.624359810566316,
|
| 653 |
+
"calls": 0.6985792473697643,
|
| 654 |
+
"gflops": 9034.097597507143,
|
| 655 |
+
"ms": 2621.0719606941566
|
| 656 |
+
},
|
| 657 |
+
{
|
| 658 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 659 |
+
"method": "learned",
|
| 660 |
+
"metric": 0.62159203697927,
|
| 661 |
+
"calls": 0.7093157133539403,
|
| 662 |
+
"gflops": 9172.771378040346,
|
| 663 |
+
"ms": 2660.4444389860882
|
| 664 |
+
},
|
| 665 |
+
{
|
| 666 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 667 |
+
"method": "learned",
|
| 668 |
+
"metric": 0.6185803817032016,
|
| 669 |
+
"calls": 0.7205569577177211,
|
| 670 |
+
"gflops": 9317.964950685257,
|
| 671 |
+
"ms": 2701.6680269283015
|
| 672 |
+
},
|
| 673 |
+
{
|
| 674 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 675 |
+
"method": "learned",
|
| 676 |
+
"metric": 0.6154042480787226,
|
| 677 |
+
"calls": 0.7306752119785611,
|
| 678 |
+
"gflops": 9448.653817283903,
|
| 679 |
+
"ms": 2738.77341587326
|
| 680 |
+
},
|
| 681 |
+
{
|
| 682 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 683 |
+
"method": "learned",
|
| 684 |
+
"metric": 0.6121713977823781,
|
| 685 |
+
"calls": 0.7409692879221847,
|
| 686 |
+
"gflops": 9581.613622707973,
|
| 687 |
+
"ms": 2776.5235733480918
|
| 688 |
+
},
|
| 689 |
+
{
|
| 690 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 691 |
+
"method": "learned",
|
| 692 |
+
"metric": 0.6087967558063693,
|
| 693 |
+
"calls": 0.7520970989422342,
|
| 694 |
+
"gflops": 9725.342073530028,
|
| 695 |
+
"ms": 2817.3311815936127
|
| 696 |
+
},
|
| 697 |
+
{
|
| 698 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 699 |
+
"method": "learned",
|
| 700 |
+
"metric": 0.6053710688256813,
|
| 701 |
+
"calls": 0.7639338683606047,
|
| 702 |
+
"gflops": 9878.227535744925,
|
| 703 |
+
"ms": 2860.73866294346
|
| 704 |
+
},
|
| 705 |
+
{
|
| 706 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 707 |
+
"method": "learned",
|
| 708 |
+
"metric": 0.6012931401185378,
|
| 709 |
+
"calls": 0.7769787028897144,
|
| 710 |
+
"gflops": 10046.716545373221,
|
| 711 |
+
"ms": 2908.576328063081
|
| 712 |
+
},
|
| 713 |
+
{
|
| 714 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 715 |
+
"method": "learned",
|
| 716 |
+
"metric": 0.5976405864503871,
|
| 717 |
+
"calls": 0.7888154723080849,
|
| 718 |
+
"gflops": 10199.602007588117,
|
| 719 |
+
"ms": 2951.9838094129286
|
| 720 |
+
},
|
| 721 |
+
{
|
| 722 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 723 |
+
"method": "learned",
|
| 724 |
+
"metric": 0.5939540027791168,
|
| 725 |
+
"calls": 0.799671043303179,
|
| 726 |
+
"gflops": 10339.81416603532,
|
| 727 |
+
"ms": 2991.793066386387
|
| 728 |
+
},
|
| 729 |
+
{
|
| 730 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 731 |
+
"method": "learned",
|
| 732 |
+
"metric": 0.5896775657204435,
|
| 733 |
+
"calls": 0.8114284093809374,
|
| 734 |
+
"gflops": 10491.674042974217,
|
| 735 |
+
"ms": 3034.90936194855
|
| 736 |
+
},
|
| 737 |
+
{
|
| 738 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 739 |
+
"method": "learned",
|
| 740 |
+
"metric": 0.5853273969883448,
|
| 741 |
+
"calls": 0.823225477129002,
|
| 742 |
+
"gflops": 10644.046712551113,
|
| 743 |
+
"ms": 3078.1712504045554
|
| 744 |
+
},
|
| 745 |
+
{
|
| 746 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 747 |
+
"method": "learned",
|
| 748 |
+
"metric": 0.5808467799109548,
|
| 749 |
+
"calls": 0.8346992598474321,
|
| 750 |
+
"gflops": 10792.243784932874,
|
| 751 |
+
"ms": 3120.2475967249875
|
| 752 |
+
},
|
| 753 |
+
{
|
| 754 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 755 |
+
"method": "learned",
|
| 756 |
+
"metric": 0.5766554178600799,
|
| 757 |
+
"calls": 0.8458951308737204,
|
| 758 |
+
"gflops": 10936.851308848642,
|
| 759 |
+
"ms": 3161.304792788524
|
| 760 |
+
},
|
| 761 |
+
{
|
| 762 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 763 |
+
"method": "learned",
|
| 764 |
+
"metric": 0.5724016674701529,
|
| 765 |
+
"calls": 0.8569038368828519,
|
| 766 |
+
"gflops": 11079.0413817567,
|
| 767 |
+
"ms": 3201.6756223525176
|
| 768 |
+
},
|
| 769 |
+
{
|
| 770 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 771 |
+
"method": "learned",
|
| 772 |
+
"metric": 0.5677111987068598,
|
| 773 |
+
"calls": 0.868638516291864,
|
| 774 |
+
"gflops": 11230.608234331025,
|
| 775 |
+
"ms": 3244.7087219753416
|
| 776 |
+
},
|
| 777 |
+
{
|
| 778 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 779 |
+
"method": "learned",
|
| 780 |
+
"metric": 0.5631284916201118,
|
| 781 |
+
"calls": 0.8793125939369878,
|
| 782 |
+
"gflops": 11368.476197861659,
|
| 783 |
+
"ms": 3283.852411434093
|
| 784 |
+
},
|
| 785 |
+
{
|
| 786 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 787 |
+
"method": "learned",
|
| 788 |
+
"metric": 0.5585287695318039,
|
| 789 |
+
"calls": 0.8898675665711936,
|
| 790 |
+
"gflops": 11504.805783478296,
|
| 791 |
+
"ms": 3322.5593222113166
|
| 792 |
+
},
|
| 793 |
+
{
|
| 794 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 795 |
+
"method": "learned",
|
| 796 |
+
"metric": 0.5539857641153618,
|
| 797 |
+
"calls": 0.9006267192241161,
|
| 798 |
+
"gflops": 11643.772588376074,
|
| 799 |
+
"ms": 3362.0149964425877
|
| 800 |
+
},
|
| 801 |
+
{
|
| 802 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 803 |
+
"method": "learned",
|
| 804 |
+
"metric": 0.5498227604004197,
|
| 805 |
+
"calls": 0.9102855684428437,
|
| 806 |
+
"gflops": 11768.527711592154,
|
| 807 |
+
"ms": 3397.435667615941
|
| 808 |
+
},
|
| 809 |
+
{
|
| 810 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 811 |
+
"method": "learned",
|
| 812 |
+
"metric": 0.5444630349091115,
|
| 813 |
+
"calls": 0.9222641295408784,
|
| 814 |
+
"gflops": 11923.244576085619,
|
| 815 |
+
"ms": 3441.363123586654
|
| 816 |
+
},
|
| 817 |
+
{
|
| 818 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 819 |
+
"method": "learned",
|
| 820 |
+
"metric": 0.539069279414684,
|
| 821 |
+
"calls": 0.9331140288687859,
|
| 822 |
+
"gflops": 12063.38347844168,
|
| 823 |
+
"ms": 3481.151581575278
|
| 824 |
+
},
|
| 825 |
+
{
|
| 826 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 827 |
+
"method": "learned",
|
| 828 |
+
"metric": 0.5336868672546294,
|
| 829 |
+
"calls": 0.9439185548592007,
|
| 830 |
+
"gflops": 12202.936332068595,
|
| 831 |
+
"ms": 3520.7736476852256
|
| 832 |
+
},
|
| 833 |
+
{
|
| 834 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 835 |
+
"method": "learned",
|
| 836 |
+
"metric": 0.5275557950259478,
|
| 837 |
+
"calls": 0.9553469642401383,
|
| 838 |
+
"gflops": 12350.547355721215,
|
| 839 |
+
"ms": 3562.683602126981
|
| 840 |
+
},
|
| 841 |
+
{
|
| 842 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 843 |
+
"method": "learned",
|
| 844 |
+
"metric": 0.520874571080169,
|
| 845 |
+
"calls": 0.966695970280464,
|
| 846 |
+
"gflops": 12497.132794097837,
|
| 847 |
+
"ms": 3604.3023707810516
|
| 848 |
+
},
|
| 849 |
+
{
|
| 850 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 851 |
+
"method": "learned",
|
| 852 |
+
"metric": 0.5142840938093752,
|
| 853 |
+
"calls": 0.977761392961461,
|
| 854 |
+
"gflops": 12640.05542791732,
|
| 855 |
+
"ms": 3644.8811901933914
|
| 856 |
+
},
|
| 857 |
+
{
|
| 858 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 859 |
+
"method": "learned",
|
| 860 |
+
"metric": 0.5079318265604174,
|
| 861 |
+
"calls": 0.9880441255707115,
|
| 862 |
+
"gflops": 12772.868721159106,
|
| 863 |
+
"ms": 3682.5897496985535
|
| 864 |
+
},
|
| 865 |
+
{
|
| 866 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 867 |
+
"method": "learned",
|
| 868 |
+
"metric": 0.5000708958398321,
|
| 869 |
+
"calls": 1.0,
|
| 870 |
+
"gflops": 12927.292561287999,
|
| 871 |
+
"ms": 3726.434009729928
|
| 872 |
+
},
|
| 873 |
+
{
|
| 874 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 875 |
+
"method": "confidence",
|
| 876 |
+
"metric": 0.6695119530385957,
|
| 877 |
+
"calls": 0.0,
|
| 878 |
+
"gflops": 7.502680064,
|
| 879 |
+
"ms": 46.08042127620138
|
| 880 |
+
},
|
| 881 |
+
{
|
| 882 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 883 |
+
"method": "confidence",
|
| 884 |
+
"metric": 0.6686044862887446,
|
| 885 |
+
"calls": 0.002467175226157729,
|
| 886 |
+
"gflops": 39.3690797110914,
|
| 887 |
+
"ms": 55.12797967926019
|
| 888 |
+
},
|
| 889 |
+
{
|
| 890 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 891 |
+
"method": "confidence",
|
| 892 |
+
"metric": 0.6678388112185577,
|
| 893 |
+
"calls": 0.004849275444516916,
|
| 894 |
+
"gflops": 70.13663799104171,
|
| 895 |
+
"ms": 63.86355330979973
|
| 896 |
+
},
|
| 897 |
+
{
|
| 898 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 899 |
+
"method": "confidence",
|
| 900 |
+
"metric": 0.667214927828035,
|
| 901 |
+
"calls": 0.007600034030003119,
|
| 902 |
+
"gflops": 105.66584219527005,
|
| 903 |
+
"ms": 73.95106095458944
|
| 904 |
+
},
|
| 905 |
+
{
|
| 906 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 907 |
+
"method": "confidence",
|
| 908 |
+
"metric": 0.6666761194453109,
|
| 909 |
+
"calls": 0.009811984232765221,
|
| 910 |
+
"gflops": 134.23571774093818,
|
| 911 |
+
"ms": 82.06266504009045
|
| 912 |
+
},
|
| 913 |
+
{
|
| 914 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 915 |
+
"method": "confidence",
|
| 916 |
+
"metric": 0.6662791027422511,
|
| 917 |
+
"calls": 0.012534384482318578,
|
| 918 |
+
"gflops": 169.39864148945287,
|
| 919 |
+
"ms": 92.04617776070705
|
| 920 |
+
},
|
| 921 |
+
{
|
| 922 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 923 |
+
"method": "confidence",
|
| 924 |
+
"metric": 0.6658537277032583,
|
| 925 |
+
"calls": 0.01607917647392451,
|
| 926 |
+
"gflops": 215.18369845366465,
|
| 927 |
+
"ms": 105.04554328234329
|
| 928 |
+
},
|
| 929 |
+
{
|
| 930 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 931 |
+
"method": "confidence",
|
| 932 |
+
"metric": 0.6654850693361314,
|
| 933 |
+
"calls": 0.019709043473328986,
|
| 934 |
+
"gflops": 262.0675967850175,
|
| 935 |
+
"ms": 118.35689357649878
|
| 936 |
+
},
|
| 937 |
+
{
|
| 938 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 939 |
+
"method": "confidence",
|
| 940 |
+
"metric": 0.6652865609846014,
|
| 941 |
+
"calls": 0.02353741882426339,
|
| 942 |
+
"gflops": 311.5154583063662,
|
| 943 |
+
"ms": 132.3962083398659
|
| 944 |
+
},
|
| 945 |
+
{
|
| 946 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 947 |
+
"method": "confidence",
|
| 948 |
+
"metric": 0.6648611859456087,
|
| 949 |
+
"calls": 0.028868785979638715,
|
| 950 |
+
"gflops": 380.3761839805407,
|
| 951 |
+
"ms": 151.94725408440678
|
| 952 |
+
},
|
| 953 |
+
{
|
| 954 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 955 |
+
"method": "confidence",
|
| 956 |
+
"metric": 0.6645208859144145,
|
| 957 |
+
"calls": 0.0345971698380739,
|
| 958 |
+
"gflops": 454.3648360347069,
|
| 959 |
+
"ms": 172.9542287673709
|
| 960 |
+
},
|
| 961 |
+
{
|
| 962 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 963 |
+
"method": "confidence",
|
| 964 |
+
"metric": 0.6642089442191532,
|
| 965 |
+
"calls": 0.03981510364971783,
|
| 966 |
+
"gflops": 521.7604398860266,
|
| 967 |
+
"ms": 192.08929481521943
|
| 968 |
+
},
|
| 969 |
+
{
|
| 970 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 971 |
+
"method": "confidence",
|
| 972 |
+
"metric": 0.6640387942035562,
|
| 973 |
+
"calls": 0.045855429203414344,
|
| 974 |
+
"gflops": 599.7781769530435,
|
| 975 |
+
"ms": 214.24021366408755
|
| 976 |
+
},
|
| 977 |
+
{
|
| 978 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 979 |
+
"method": "confidence",
|
| 980 |
+
"metric": 0.6637268525082948,
|
| 981 |
+
"calls": 0.05317187987408899,
|
| 982 |
+
"gflops": 694.2785345271766,
|
| 983 |
+
"ms": 241.0709041007447
|
| 984 |
+
},
|
| 985 |
+
{
|
| 986 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 987 |
+
"method": "confidence",
|
| 988 |
+
"metric": 0.6633581941411678,
|
| 989 |
+
"calls": 0.05960922213084536,
|
| 990 |
+
"gflops": 777.4241979741852,
|
| 991 |
+
"ms": 264.6777518880361
|
| 992 |
+
},
|
| 993 |
+
{
|
| 994 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 995 |
+
"method": "confidence",
|
| 996 |
+
"metric": 0.6630178941099736,
|
| 997 |
+
"calls": 0.0675211978561098,
|
| 998 |
+
"gflops": 879.6164451183058,
|
| 999 |
+
"ms": 293.69233573232816
|
| 1000 |
+
},
|
| 1001 |
+
{
|
| 1002 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1003 |
+
"method": "confidence",
|
| 1004 |
+
"metric": 0.6626492357428466,
|
| 1005 |
+
"calls": 0.07551824858917279,
|
| 1006 |
+
"gflops": 982.9075336295676,
|
| 1007 |
+
"ms": 323.01890434913946
|
| 1008 |
+
},
|
| 1009 |
+
{
|
| 1010 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1011 |
+
"method": "confidence",
|
| 1012 |
+
"metric": 0.6621671440319882,
|
| 1013 |
+
"calls": 0.08212574086152624,
|
| 1014 |
+
"gflops": 1068.2508798108586,
|
| 1015 |
+
"ms": 347.24972168146934
|
| 1016 |
+
},
|
| 1017 |
+
{
|
| 1018 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1019 |
+
"method": "confidence",
|
| 1020 |
+
"metric": 0.6619969940163911,
|
| 1021 |
+
"calls": 0.08856308311828262,
|
| 1022 |
+
"gflops": 1151.396543257867,
|
| 1023 |
+
"ms": 370.85656946876077
|
| 1024 |
+
},
|
| 1025 |
+
{
|
| 1026 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1027 |
+
"method": "confidence",
|
| 1028 |
+
"metric": 0.6615716189773984,
|
| 1029 |
+
"calls": 0.09480191702350906,
|
| 1030 |
+
"gflops": 1231.9782435148798,
|
| 1031 |
+
"ms": 393.7354527868405
|
| 1032 |
+
},
|
| 1033 |
+
{
|
| 1034 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1035 |
+
"method": "confidence",
|
| 1036 |
+
"metric": 0.6612029606102714,
|
| 1037 |
+
"calls": 0.10007656750701868,
|
| 1038 |
+
"gflops": 1300.1064082776268,
|
| 1039 |
+
"ms": 413.0785086830352
|
| 1040 |
+
},
|
| 1041 |
+
{
|
| 1042 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1043 |
+
"method": "confidence",
|
| 1044 |
+
"metric": 0.6608910189150101,
|
| 1045 |
+
"calls": 0.10554972634205825,
|
| 1046 |
+
"gflops": 1370.79853623037,
|
| 1047 |
+
"ms": 433.14952904844154
|
| 1048 |
+
},
|
| 1049 |
+
{
|
| 1050 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1051 |
+
"method": "confidence",
|
| 1052 |
+
"metric": 0.6605790772197487,
|
| 1053 |
+
"calls": 0.11005870175538099,
|
| 1054 |
+
"gflops": 1429.0371286888474,
|
| 1055 |
+
"ms": 449.68472199196276
|
| 1056 |
+
},
|
| 1057 |
+
{
|
| 1058 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1059 |
+
"method": "confidence",
|
| 1060 |
+
"metric": 0.6602671355244875,
|
| 1061 |
+
"calls": 0.11527663556702493,
|
| 1062 |
+
"gflops": 1496.432732540167,
|
| 1063 |
+
"ms": 468.8197880398113
|
| 1064 |
+
},
|
| 1065 |
+
{
|
| 1066 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1067 |
+
"method": "confidence",
|
| 1068 |
+
"metric": 0.6599835521651589,
|
| 1069 |
+
"calls": 0.12029606102713893,
|
| 1070 |
+
"gflops": 1561.264373201491,
|
| 1071 |
+
"ms": 487.2268896184482
|
| 1072 |
+
},
|
| 1073 |
+
{
|
| 1074 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1075 |
+
"method": "confidence",
|
| 1076 |
+
"metric": 0.6596999688058305,
|
| 1077 |
+
"calls": 0.12477667810452883,
|
| 1078 |
+
"gflops": 1619.1366852042547,
|
| 1079 |
+
"ms": 503.6580876377964
|
| 1080 |
+
},
|
| 1081 |
+
{
|
| 1082 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1083 |
+
"method": "confidence",
|
| 1084 |
+
"metric": 0.659217877094972,
|
| 1085 |
+
"calls": 0.12914386183818732,
|
| 1086 |
+
"gflops": 1675.5438753841634,
|
| 1087 |
+
"ms": 519.6733059604521
|
| 1088 |
+
},
|
| 1089 |
+
{
|
| 1090 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1091 |
+
"method": "confidence",
|
| 1092 |
+
"metric": 0.6587925020559794,
|
| 1093 |
+
"calls": 0.1322916371267334,
|
| 1094 |
+
"gflops": 1716.2010059683835,
|
| 1095 |
+
"ms": 531.2167425436651
|
| 1096 |
+
},
|
| 1097 |
+
{
|
| 1098 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1099 |
+
"method": "confidence",
|
| 1100 |
+
"metric": 0.6582253353373224,
|
| 1101 |
+
"calls": 0.13626180415733205,
|
| 1102 |
+
"gflops": 1767.4802697683008,
|
| 1103 |
+
"ms": 545.7760319278977
|
| 1104 |
+
},
|
| 1105 |
+
{
|
| 1106 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1107 |
+
"method": "confidence",
|
| 1108 |
+
"metric": 0.6578283186342625,
|
| 1109 |
+
"calls": 0.14023197118793068,
|
| 1110 |
+
"gflops": 1818.7595335682179,
|
| 1111 |
+
"ms": 560.3353213121303
|
| 1112 |
+
},
|
| 1113 |
+
{
|
| 1114 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1115 |
+
"method": "confidence",
|
| 1116 |
+
"metric": 0.6573178685874713,
|
| 1117 |
+
"calls": 0.14326631313274538,
|
| 1118 |
+
"gflops": 1857.9515423295834,
|
| 1119 |
+
"ms": 571.462778198651
|
| 1120 |
+
},
|
| 1121 |
+
{
|
| 1122 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1123 |
+
"method": "confidence",
|
| 1124 |
+
"metric": 0.6569208518844114,
|
| 1125 |
+
"calls": 0.14766185520233674,
|
| 1126 |
+
"gflops": 1914.725012965206,
|
| 1127 |
+
"ms": 587.5819914454798
|
| 1128 |
+
},
|
| 1129 |
+
{
|
| 1130 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1131 |
+
"method": "confidence",
|
| 1132 |
+
"metric": 0.6565805518532173,
|
| 1133 |
+
"calls": 0.15066783881121856,
|
| 1134 |
+
"gflops": 1953.5507412708575,
|
| 1135 |
+
"ms": 598.6054534078274
|
| 1136 |
+
},
|
| 1137 |
+
{
|
| 1138 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1139 |
+
"method": "confidence",
|
| 1140 |
+
"metric": 0.6561268184782917,
|
| 1141 |
+
"calls": 0.15452457249808582,
|
| 1142 |
+
"gflops": 2003.36488324792,
|
| 1143 |
+
"ms": 612.7487630953676
|
| 1144 |
+
},
|
| 1145 |
+
{
|
| 1146 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1147 |
+
"method": "confidence",
|
| 1148 |
+
"metric": 0.6558715934548961,
|
| 1149 |
+
"calls": 0.15806936448969175,
|
| 1150 |
+
"gflops": 2049.1499402121317,
|
| 1151 |
+
"ms": 625.7481286170038
|
| 1152 |
+
},
|
| 1153 |
+
{
|
| 1154 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1155 |
+
"method": "confidence",
|
| 1156 |
+
"metric": 0.6555596517596347,
|
| 1157 |
+
"calls": 0.16201117318435754,
|
| 1158 |
+
"gflops": 2100.0629235563347,
|
| 1159 |
+
"ms": 640.2034230770632
|
| 1160 |
+
},
|
| 1161 |
+
{
|
| 1162 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1163 |
+
"method": "confidence",
|
| 1164 |
+
"metric": 0.6551626350565749,
|
| 1165 |
+
"calls": 0.1657544735274934,
|
| 1166 |
+
"gflops": 2148.4119437105423,
|
| 1167 |
+
"ms": 653.930753067911
|
| 1168 |
+
},
|
| 1169 |
+
{
|
| 1170 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1171 |
+
"method": "confidence",
|
| 1172 |
+
"metric": 0.6545954683379179,
|
| 1173 |
+
"calls": 0.17012165726115192,
|
| 1174 |
+
"gflops": 2204.8191338904517,
|
| 1175 |
+
"ms": 669.9459713905669
|
| 1176 |
+
},
|
| 1177 |
+
{
|
| 1178 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1179 |
+
"method": "confidence",
|
| 1180 |
+
"metric": 0.6540850182911266,
|
| 1181 |
+
"calls": 0.17536794940872868,
|
| 1182 |
+
"gflops": 2272.581018197485,
|
| 1183 |
+
"ms": 689.1850323625886
|
| 1184 |
+
},
|
| 1185 |
+
{
|
| 1186 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1187 |
+
"method": "confidence",
|
| 1188 |
+
"metric": 0.6534044182287383,
|
| 1189 |
+
"calls": 0.17965005813458867,
|
| 1190 |
+
"gflops": 2327.889367010253,
|
| 1191 |
+
"ms": 704.8882659127252
|
| 1192 |
+
},
|
| 1193 |
+
{
|
| 1194 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1195 |
+
"method": "confidence",
|
| 1196 |
+
"metric": 0.653092476533477,
|
| 1197 |
+
"calls": 0.18401724186824717,
|
| 1198 |
+
"gflops": 2384.2965571901614,
|
| 1199 |
+
"ms": 720.9034842353809
|
| 1200 |
+
},
|
| 1201 |
+
{
|
| 1202 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1203 |
+
"method": "confidence",
|
| 1204 |
+
"metric": 0.6527521765022829,
|
| 1205 |
+
"calls": 0.18858293395343562,
|
| 1206 |
+
"gflops": 2443.2677105600665,
|
| 1207 |
+
"ms": 737.6466670272484
|
| 1208 |
+
},
|
| 1209 |
+
{
|
| 1210 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1211 |
+
"method": "confidence",
|
| 1212 |
+
"metric": 0.6523835181351558,
|
| 1213 |
+
"calls": 0.19238295096843716,
|
| 1214 |
+
"gflops": 2492.3492916257014,
|
| 1215 |
+
"ms": 751.5819868664424
|
| 1216 |
+
},
|
| 1217 |
+
{
|
| 1218 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1219 |
+
"method": "confidence",
|
| 1220 |
+
"metric": 0.6517596347446332,
|
| 1221 |
+
"calls": 0.19689192638175992,
|
| 1222 |
+
"gflops": 2550.5878840841788,
|
| 1223 |
+
"ms": 768.1171798099638
|
| 1224 |
+
},
|
| 1225 |
+
{
|
| 1226 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1227 |
+
"method": "confidence",
|
| 1228 |
+
"metric": 0.6511641096900433,
|
| 1229 |
+
"calls": 0.20182627683407536,
|
| 1230 |
+
"gflops": 2614.3206833783615,
|
| 1231 |
+
"ms": 786.2122966160813
|
| 1232 |
+
},
|
| 1233 |
+
{
|
| 1234 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1235 |
+
"method": "confidence",
|
| 1236 |
+
"metric": 0.6505402262995208,
|
| 1237 |
+
"calls": 0.20718600232538353,
|
| 1238 |
+
"gflops": 2683.5476895082497,
|
| 1239 |
+
"ms": 805.8673372847952
|
| 1240 |
+
},
|
| 1241 |
+
{
|
| 1242 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1243 |
+
"method": "confidence",
|
| 1244 |
+
"metric": 0.6500864929245952,
|
| 1245 |
+
"calls": 0.21189348609023623,
|
| 1246 |
+
"gflops": 2744.350245156723,
|
| 1247 |
+
"ms": 823.1304946975282
|
| 1248 |
+
},
|
| 1249 |
+
{
|
| 1250 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1251 |
+
"method": "confidence",
|
| 1252 |
+
"metric": 0.6500581345886624,
|
| 1253 |
+
"calls": 0.217139778237813,
|
| 1254 |
+
"gflops": 2812.1121294637564,
|
| 1255 |
+
"ms": 842.3695556695498
|
| 1256 |
+
},
|
| 1257 |
+
{
|
| 1258 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1259 |
+
"method": "confidence",
|
| 1260 |
+
"metric": 0.6507670929869835,
|
| 1261 |
+
"calls": 0.22198905368232993,
|
| 1262 |
+
"gflops": 2874.7460873907985,
|
| 1263 |
+
"ms": 860.1526877031482
|
| 1264 |
+
},
|
| 1265 |
+
{
|
| 1266 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1267 |
+
"method": "confidence",
|
| 1268 |
+
"metric": 0.6515611263931033,
|
| 1269 |
+
"calls": 0.22763236253296656,
|
| 1270 |
+
"gflops": 2947.635898077823,
|
| 1271 |
+
"ms": 880.8476776135931
|
| 1272 |
+
},
|
| 1273 |
+
{
|
| 1274 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1275 |
+
"method": "confidence",
|
| 1276 |
+
"metric": 0.6527521765022829,
|
| 1277 |
+
"calls": 0.2345234381646485,
|
| 1278 |
+
"gflops": 3036.6420488162507,
|
| 1279 |
+
"ms": 906.1184441876538
|
| 1280 |
+
},
|
| 1281 |
+
{
|
| 1282 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1283 |
+
"method": "confidence",
|
| 1284 |
+
"metric": 0.6537447182599325,
|
| 1285 |
+
"calls": 0.24382497235062248,
|
| 1286 |
+
"gflops": 3156.782038290343,
|
| 1287 |
+
"ms": 940.2287793164273
|
| 1288 |
+
},
|
| 1289 |
+
{
|
| 1290 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1291 |
+
"method": "confidence",
|
| 1292 |
+
"metric": 0.6545387516660522,
|
| 1293 |
+
"calls": 0.252644414825738,
|
| 1294 |
+
"gflops": 3270.6952600173017,
|
| 1295 |
+
"ms": 972.5712007342581
|
| 1296 |
+
},
|
| 1297 |
+
{
|
| 1298 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1299 |
+
"method": "confidence",
|
| 1300 |
+
"metric": 0.6538581516036639,
|
| 1301 |
+
"calls": 0.26401610753480986,
|
| 1302 |
+
"gflops": 3417.573722758493,
|
| 1303 |
+
"ms": 1014.2731653276672
|
| 1304 |
+
},
|
| 1305 |
+
{
|
| 1306 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1307 |
+
"method": "confidence",
|
| 1308 |
+
"metric": 0.6537163599239997,
|
| 1309 |
+
"calls": 0.2760116836344043,
|
| 1310 |
+
"gflops": 3572.5103555253854,
|
| 1311 |
+
"ms": 1058.2630182528842
|
| 1312 |
+
},
|
| 1313 |
+
{
|
| 1314 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1315 |
+
"method": "confidence",
|
| 1316 |
+
"metric": 0.6533477015568726,
|
| 1317 |
+
"calls": 0.2893401015228426,
|
| 1318 |
+
"gflops": 3744.6621697108217,
|
| 1319 |
+
"ms": 1107.1406326142362
|
| 1320 |
+
},
|
| 1321 |
+
{
|
| 1322 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1323 |
+
"method": "confidence",
|
| 1324 |
+
"metric": 0.6525536681507529,
|
| 1325 |
+
"calls": 0.3036043444970649,
|
| 1326 |
+
"gflops": 3928.90123893481,
|
| 1327 |
+
"ms": 1159.4500794733005
|
| 1328 |
+
},
|
| 1329 |
+
{
|
| 1330 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1331 |
+
"method": "confidence",
|
| 1332 |
+
"metric": 0.6509088846666478,
|
| 1333 |
+
"calls": 0.3175850041119587,
|
| 1334 |
+
"gflops": 4109.477503601662,
|
| 1335 |
+
"ms": 1210.719577090634
|
| 1336 |
+
},
|
| 1337 |
+
{
|
| 1338 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1339 |
+
"method": "confidence",
|
| 1340 |
+
"metric": 0.6498312679011996,
|
| 1341 |
+
"calls": 0.33462836400760004,
|
| 1342 |
+
"gflops": 4329.612057485591,
|
| 1343 |
+
"ms": 1273.2205265186608
|
| 1344 |
+
},
|
| 1345 |
+
{
|
| 1346 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1347 |
+
"method": "confidence",
|
| 1348 |
+
"metric": 0.6465984176048549,
|
| 1349 |
+
"calls": 0.35637920766809406,
|
| 1350 |
+
"gflops": 4610.549167017995,
|
| 1351 |
+
"ms": 1352.9846333594207
|
| 1352 |
+
},
|
| 1353 |
+
{
|
| 1354 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1355 |
+
"method": "confidence",
|
| 1356 |
+
"metric": 0.6442730340583614,
|
| 1357 |
+
"calls": 0.37968975980489467,
|
| 1358 |
+
"gflops": 4911.631701614652,
|
| 1359 |
+
"ms": 1438.4684610297004
|
| 1360 |
+
},
|
| 1361 |
+
{
|
| 1362 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1363 |
+
"method": "confidence",
|
| 1364 |
+
"metric": 0.6405580920511584,
|
| 1365 |
+
"calls": 0.4046450954258004,
|
| 1366 |
+
"gflops": 5233.958502642702,
|
| 1367 |
+
"ms": 1529.9839943020195
|
| 1368 |
+
},
|
| 1369 |
+
{
|
| 1370 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1371 |
+
"method": "confidence",
|
| 1372 |
+
"metric": 0.6357371749425744,
|
| 1373 |
+
"calls": 0.43141536454640844,
|
| 1374 |
+
"gflops": 5579.72725283643,
|
| 1375 |
+
"ms": 1628.1552027214163
|
| 1376 |
+
},
|
| 1377 |
+
{
|
| 1378 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1379 |
+
"method": "confidence",
|
| 1380 |
+
"metric": 0.630604316138729,
|
| 1381 |
+
"calls": 0.4610214672603012,
|
| 1382 |
+
"gflops": 5962.124048601527,
|
| 1383 |
+
"ms": 1736.725903558122
|
| 1384 |
+
},
|
| 1385 |
+
{
|
| 1386 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1387 |
+
"method": "confidence",
|
| 1388 |
+
"metric": 0.6244221989053682,
|
| 1389 |
+
"calls": 0.4916484700677764,
|
| 1390 |
+
"gflops": 6357.706940772317,
|
| 1391 |
+
"ms": 1849.0404216650588
|
| 1392 |
+
},
|
| 1393 |
+
{
|
| 1394 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1395 |
+
"method": "confidence",
|
| 1396 |
+
"metric": 0.6187788900547316,
|
| 1397 |
+
"calls": 0.5209426310864078,
|
| 1398 |
+
"gflops": 6736.074651524562,
|
| 1399 |
+
"ms": 1956.4671783358606
|
| 1400 |
+
},
|
| 1401 |
+
{
|
| 1402 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1403 |
+
"method": "confidence",
|
| 1404 |
+
"metric": 0.6118310977511839,
|
| 1405 |
+
"calls": 0.5529308340186598,
|
| 1406 |
+
"gflops": 7149.2390055696105,
|
| 1407 |
+
"ms": 2073.773452803106
|
| 1408 |
+
},
|
| 1409 |
+
{
|
| 1410 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1411 |
+
"method": "confidence",
|
| 1412 |
+
"metric": 0.6045146470805093,
|
| 1413 |
+
"calls": 0.5888041289737118,
|
| 1414 |
+
"gflops": 7612.583782047433,
|
| 1415 |
+
"ms": 2205.3270318820646
|
| 1416 |
+
},
|
| 1417 |
+
{
|
| 1418 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1419 |
+
"method": "confidence",
|
| 1420 |
+
"metric": 0.5983608881830814,
|
| 1421 |
+
"calls": 0.6175027649377535,
|
| 1422 |
+
"gflops": 7983.2596032296915,
|
| 1423 |
+
"ms": 2310.5698951452314
|
| 1424 |
+
},
|
| 1425 |
+
{
|
| 1426 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1427 |
+
"method": "confidence",
|
| 1428 |
+
"metric": 0.5925474293168477,
|
| 1429 |
+
"calls": 0.6421461588633979,
|
| 1430 |
+
"gflops": 8301.557319244892,
|
| 1431 |
+
"ms": 2400.941484251646
|
| 1432 |
+
},
|
| 1433 |
+
{
|
| 1434 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1435 |
+
"method": "confidence",
|
| 1436 |
+
"metric": 0.5877548705441965,
|
| 1437 |
+
"calls": 0.665910444375124,
|
| 1438 |
+
"gflops": 8608.500341132967,
|
| 1439 |
+
"ms": 2488.0892307086956
|
| 1440 |
+
},
|
| 1441 |
+
{
|
| 1442 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1443 |
+
"method": "confidence",
|
| 1444 |
+
"metric": 0.5829339534356124,
|
| 1445 |
+
"calls": 0.6838612710206166,
|
| 1446 |
+
"gflops": 8840.355869599738,
|
| 1447 |
+
"ms": 2553.9180177102617
|
| 1448 |
+
},
|
| 1449 |
+
{
|
| 1450 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1451 |
+
"method": "confidence",
|
| 1452 |
+
"metric": 0.5779996029832969,
|
| 1453 |
+
"calls": 0.7029464311034228,
|
| 1454 |
+
"gflops": 9086.862616295053,
|
| 1455 |
+
"ms": 2623.9066016787506
|
| 1456 |
+
},
|
| 1457 |
+
{
|
| 1458 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1459 |
+
"method": "confidence",
|
| 1460 |
+
"metric": 0.5745966026713553,
|
| 1461 |
+
"calls": 0.7180614241556306,
|
| 1462 |
+
"gflops": 9282.090099190453,
|
| 1463 |
+
"ms": 2679.3358962630077
|
| 1464 |
+
},
|
| 1465 |
+
{
|
| 1466 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1467 |
+
"method": "confidence",
|
| 1468 |
+
"metric": 0.5710234523438165,
|
| 1469 |
+
"calls": 0.7352465757309361,
|
| 1470 |
+
"gflops": 9504.05605535295,
|
| 1471 |
+
"ms": 2742.3568203119
|
| 1472 |
+
},
|
| 1473 |
+
{
|
| 1474 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1475 |
+
"method": "confidence",
|
| 1476 |
+
"metric": 0.567762243711539,
|
| 1477 |
+
"calls": 0.7478093185491875,
|
| 1478 |
+
"gflops": 9666.318297234116,
|
| 1479 |
+
"ms": 2788.426571720579
|
| 1480 |
+
},
|
| 1481 |
+
{
|
| 1482 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1483 |
+
"method": "confidence",
|
| 1484 |
+
"metric": 0.565153276805717,
|
| 1485 |
+
"calls": 0.759804894648782,
|
| 1486 |
+
"gflops": 9821.254930001009,
|
| 1487 |
+
"ms": 2832.416424645796
|
| 1488 |
+
},
|
| 1489 |
+
{
|
| 1490 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1491 |
+
"method": "confidence",
|
| 1492 |
+
"metric": 0.5620622181890367,
|
| 1493 |
+
"calls": 0.7729064458497575,
|
| 1494 |
+
"gflops": 9990.476500540737,
|
| 1495 |
+
"ms": 2880.4620796137633
|
| 1496 |
+
},
|
| 1497 |
+
{
|
| 1498 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1499 |
+
"method": "confidence",
|
| 1500 |
+
"metric": 0.5588860845645578,
|
| 1501 |
+
"calls": 0.7855259053398747,
|
| 1502 |
+
"gflops": 10153.47130333333,
|
| 1503 |
+
"ms": 2926.739820870788
|
| 1504 |
+
},
|
| 1505 |
+
{
|
| 1506 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1507 |
+
"method": "confidence",
|
| 1508 |
+
"metric": 0.5562771176587358,
|
| 1509 |
+
"calls": 0.7964722230099538,
|
| 1510 |
+
"gflops": 10294.855559238817,
|
| 1511 |
+
"ms": 2966.881861601601
|
| 1512 |
+
},
|
| 1513 |
+
{
|
| 1514 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1515 |
+
"method": "confidence",
|
| 1516 |
+
"metric": 0.5536114340810481,
|
| 1517 |
+
"calls": 0.8082125740861527,
|
| 1518 |
+
"gflops": 10446.495667904286,
|
| 1519 |
+
"ms": 3009.93576020926
|
| 1520 |
+
},
|
| 1521 |
+
{
|
| 1522 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1523 |
+
"method": "confidence",
|
| 1524 |
+
"metric": 0.5504920171284349,
|
| 1525 |
+
"calls": 0.8197544168108215,
|
| 1526 |
+
"gflops": 10595.57181337976,
|
| 1527 |
+
"ms": 3052.2616943477074
|
| 1528 |
+
},
|
| 1529 |
+
{
|
| 1530 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1531 |
+
"method": "confidence",
|
| 1532 |
+
"metric": 0.5469188668008961,
|
| 1533 |
+
"calls": 0.8333380597226555,
|
| 1534 |
+
"gflops": 10771.02015166662,
|
| 1535 |
+
"ms": 3102.075263026618
|
| 1536 |
+
},
|
| 1537 |
+
{
|
| 1538 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1539 |
+
"method": "confidence",
|
| 1540 |
+
"metric": 0.5450188582933954,
|
| 1541 |
+
"calls": 0.8426112355726966,
|
| 1542 |
+
"gflops": 10890.793860684997,
|
| 1543 |
+
"ms": 3136.0816032312177
|
| 1544 |
+
},
|
| 1545 |
+
{
|
| 1546 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1547 |
+
"method": "confidence",
|
| 1548 |
+
"metric": 0.5423531747157077,
|
| 1549 |
+
"calls": 0.8529903865241187,
|
| 1550 |
+
"gflops": 11024.85250747621,
|
| 1551 |
+
"ms": 3174.1437454785687
|
| 1552 |
+
},
|
| 1553 |
+
{
|
| 1554 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1555 |
+
"method": "confidence",
|
| 1556 |
+
"metric": 0.5395456994583557,
|
| 1557 |
+
"calls": 0.8635396874911381,
|
| 1558 |
+
"gflops": 11161.108837001704,
|
| 1559 |
+
"ms": 3212.829857270958
|
| 1560 |
+
},
|
| 1561 |
+
{
|
| 1562 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1563 |
+
"method": "confidence",
|
| 1564 |
+
"metric": 0.5378725576383178,
|
| 1565 |
+
"calls": 0.8710262881774098,
|
| 1566 |
+
"gflops": 11257.806877310119,
|
| 1567 |
+
"ms": 3240.284517252654
|
| 1568 |
+
},
|
| 1569 |
+
{
|
| 1570 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1571 |
+
"method": "confidence",
|
| 1572 |
+
"metric": 0.535774040779287,
|
| 1573 |
+
"calls": 0.8809517057539064,
|
| 1574 |
+
"gflops": 11386.005036809911,
|
| 1575 |
+
"ms": 3276.6827407132355
|
| 1576 |
+
},
|
| 1577 |
+
{
|
| 1578 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1579 |
+
"method": "confidence",
|
| 1580 |
+
"metric": 0.5330516405297338,
|
| 1581 |
+
"calls": 0.8913024983693957,
|
| 1582 |
+
"gflops": 11519.69740314541,
|
| 1583 |
+
"ms": 3314.640888036413
|
| 1584 |
+
},
|
| 1585 |
+
{
|
| 1586 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1587 |
+
"method": "confidence",
|
| 1588 |
+
"metric": 0.5304710319598446,
|
| 1589 |
+
"calls": 0.900915974250631,
|
| 1590 |
+
"gflops": 11643.866477632353,
|
| 1591 |
+
"ms": 3349.8951673310903
|
| 1592 |
+
},
|
| 1593 |
+
{
|
| 1594 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1595 |
+
"method": "confidence",
|
| 1596 |
+
"metric": 0.5277486317102913,
|
| 1597 |
+
"calls": 0.910954825170859,
|
| 1598 |
+
"gflops": 11773.529758955001,
|
| 1599 |
+
"ms": 3386.7093704883646
|
| 1600 |
+
},
|
| 1601 |
+
{
|
| 1602 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1603 |
+
"method": "confidence",
|
| 1604 |
+
"metric": 0.5245724980858123,
|
| 1605 |
+
"calls": 0.921901142840938,
|
| 1606 |
+
"gflops": 11914.914014860486,
|
| 1607 |
+
"ms": 3426.851411219177
|
| 1608 |
+
},
|
| 1609 |
+
{
|
| 1610 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1611 |
+
"method": "confidence",
|
| 1612 |
+
"metric": 0.5217366644925275,
|
| 1613 |
+
"calls": 0.9303235686129938,
|
| 1614 |
+
"gflops": 12023.699310207454,
|
| 1615 |
+
"ms": 3457.737903698585
|
| 1616 |
+
},
|
| 1617 |
+
{
|
| 1618 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1619 |
+
"method": "confidence",
|
| 1620 |
+
"metric": 0.5187306808836457,
|
| 1621 |
+
"calls": 0.9393698777755721,
|
| 1622 |
+
"gflops": 12140.542775580121,
|
| 1623 |
+
"ms": 3490.9122845098004
|
| 1624 |
+
},
|
| 1625 |
+
{
|
| 1626 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1627 |
+
"method": "confidence",
|
| 1628 |
+
"metric": 0.515526188923234,
|
| 1629 |
+
"calls": 0.9490117119927403,
|
| 1630 |
+
"gflops": 12265.078130522777,
|
| 1631 |
+
"ms": 3526.270558728651
|
| 1632 |
+
},
|
| 1633 |
+
{
|
| 1634 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1635 |
+
"method": "confidence",
|
| 1636 |
+
"metric": 0.5122366219550237,
|
| 1637 |
+
"calls": 0.9587953378895726,
|
| 1638 |
+
"gflops": 12391.444887744003,
|
| 1639 |
+
"ms": 3562.1488075683665
|
| 1640 |
+
},
|
| 1641 |
+
{
|
| 1642 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1643 |
+
"method": "confidence",
|
| 1644 |
+
"metric": 0.5097694467288659,
|
| 1645 |
+
"calls": 0.9673595553412926,
|
| 1646 |
+
"gflops": 12502.061585369538,
|
| 1647 |
+
"ms": 3593.55527466864
|
| 1648 |
+
},
|
| 1649 |
+
{
|
| 1650 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1651 |
+
"method": "confidence",
|
| 1652 |
+
"metric": 0.5075574965261038,
|
| 1653 |
+
"calls": 0.9746192893401016,
|
| 1654 |
+
"gflops": 12595.829382032243,
|
| 1655 |
+
"ms": 3620.177975256951
|
| 1656 |
+
},
|
| 1657 |
+
{
|
| 1658 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1659 |
+
"method": "confidence",
|
| 1660 |
+
"metric": 0.5046365879250205,
|
| 1661 |
+
"calls": 0.9830984317840229,
|
| 1662 |
+
"gflops": 12705.347238290638,
|
| 1663 |
+
"ms": 3651.2724575847046
|
| 1664 |
+
},
|
| 1665 |
+
{
|
| 1666 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1667 |
+
"method": "confidence",
|
| 1668 |
+
"metric": 0.5019425460114001,
|
| 1669 |
+
"calls": 0.9908118991577575,
|
| 1670 |
+
"gflops": 12804.975522244764,
|
| 1671 |
+
"ms": 3679.559076959785
|
| 1672 |
+
},
|
| 1673 |
+
{
|
| 1674 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 1675 |
+
"method": "confidence",
|
| 1676 |
+
"metric": 0.5000708958398321,
|
| 1677 |
+
"calls": 1.0,
|
| 1678 |
+
"gflops": 12923.650389896,
|
| 1679 |
+
"ms": 3713.253432391866
|
| 1680 |
+
},
|
| 1681 |
+
{
|
| 1682 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1683 |
+
"method": "learned",
|
| 1684 |
+
"metric": 0.18129624718318374,
|
| 1685 |
+
"calls": 0.0,
|
| 1686 |
+
"gflops": 139.16228659200002,
|
| 1687 |
+
"ms": 71.56936161300109
|
| 1688 |
+
},
|
| 1689 |
+
{
|
| 1690 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1691 |
+
"method": "learned",
|
| 1692 |
+
"metric": 0.18372855448704165,
|
| 1693 |
+
"calls": 0.007789613848202396,
|
| 1694 |
+
"gflops": 893.7788890248893,
|
| 1695 |
+
"ms": 148.08265569458447
|
| 1696 |
+
},
|
| 1697 |
+
{
|
| 1698 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1699 |
+
"method": "learned",
|
| 1700 |
+
"metric": 0.18685908224889786,
|
| 1701 |
+
"calls": 0.018908122503328895,
|
| 1702 |
+
"gflops": 1970.8812189932014,
|
| 1703 |
+
"ms": 257.29393869992145
|
| 1704 |
+
},
|
| 1705 |
+
{
|
| 1706 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1707 |
+
"method": "learned",
|
| 1708 |
+
"metric": 0.1884468119835047,
|
| 1709 |
+
"calls": 0.029227696404793613,
|
| 1710 |
+
"gflops": 2970.5869743530125,
|
| 1711 |
+
"ms": 358.6577043635576
|
| 1712 |
+
},
|
| 1713 |
+
{
|
| 1714 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1715 |
+
"method": "learned",
|
| 1716 |
+
"metric": 0.19016333859244455,
|
| 1717 |
+
"calls": 0.037150466045272966,
|
| 1718 |
+
"gflops": 3738.1030058873175,
|
| 1719 |
+
"ms": 436.47891800209106
|
| 1720 |
+
},
|
| 1721 |
+
{
|
| 1722 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1723 |
+
"method": "learned",
|
| 1724 |
+
"metric": 0.19203936879648048,
|
| 1725 |
+
"calls": 0.047203728362183756,
|
| 1726 |
+
"gflops": 4712.009903044295,
|
| 1727 |
+
"ms": 535.226844551827
|
| 1728 |
+
},
|
| 1729 |
+
{
|
| 1730 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1731 |
+
"method": "learned",
|
| 1732 |
+
"metric": 0.1973276972772499,
|
| 1733 |
+
"calls": 0.06245006657789613,
|
| 1734 |
+
"gflops": 6188.99453515653,
|
| 1735 |
+
"ms": 684.9836338226183
|
| 1736 |
+
},
|
| 1737 |
+
{
|
| 1738 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1739 |
+
"method": "learned",
|
| 1740 |
+
"metric": 0.19856778453770563,
|
| 1741 |
+
"calls": 0.07423435419440745,
|
| 1742 |
+
"gflops": 7330.594010631928,
|
| 1743 |
+
"ms": 800.734514612706
|
| 1744 |
+
},
|
| 1745 |
+
{
|
| 1746 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1747 |
+
"method": "learned",
|
| 1748 |
+
"metric": 0.19964444666304987,
|
| 1749 |
+
"calls": 0.08342210386151798,
|
| 1750 |
+
"gflops": 8220.654618629695,
|
| 1751 |
+
"ms": 890.980964042266
|
| 1752 |
+
},
|
| 1753 |
+
{
|
| 1754 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1755 |
+
"method": "learned",
|
| 1756 |
+
"metric": 0.2008447299915772,
|
| 1757 |
+
"calls": 0.09094540612516644,
|
| 1758 |
+
"gflops": 8949.47236285975,
|
| 1759 |
+
"ms": 964.878419009949
|
| 1760 |
+
},
|
| 1761 |
+
{
|
| 1762 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1763 |
+
"method": "learned",
|
| 1764 |
+
"metric": 0.2053760837500663,
|
| 1765 |
+
"calls": 0.10252996005326231,
|
| 1766 |
+
"gflops": 10071.722694683021,
|
| 1767 |
+
"ms": 1078.6674204646115
|
| 1768 |
+
},
|
| 1769 |
+
{
|
| 1770 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1771 |
+
"method": "learned",
|
| 1772 |
+
"metric": 0.2084699431938867,
|
| 1773 |
+
"calls": 0.11284953395472704,
|
| 1774 |
+
"gflops": 11071.428450042833,
|
| 1775 |
+
"ms": 1180.0311861282478
|
| 1776 |
+
},
|
| 1777 |
+
{
|
| 1778 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1779 |
+
"method": "learned",
|
| 1780 |
+
"metric": 0.21117730070554924,
|
| 1781 |
+
"calls": 0.12343541944074565,
|
| 1782 |
+
"gflops": 12096.933063605475,
|
| 1783 |
+
"ms": 1284.0107909057838
|
| 1784 |
+
},
|
| 1785 |
+
{
|
| 1786 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1787 |
+
"method": "learned",
|
| 1788 |
+
"metric": 0.21355089822347736,
|
| 1789 |
+
"calls": 0.1318908122503329,
|
| 1790 |
+
"gflops": 12916.04681154545,
|
| 1791 |
+
"ms": 1367.0636827721182
|
| 1792 |
+
},
|
| 1793 |
+
{
|
| 1794 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1795 |
+
"method": "learned",
|
| 1796 |
+
"metric": 0.21604151501630087,
|
| 1797 |
+
"calls": 0.14500665778961386,
|
| 1798 |
+
"gflops": 14186.640578035016,
|
| 1799 |
+
"ms": 1495.8937591317074
|
| 1800 |
+
},
|
| 1801 |
+
{
|
| 1802 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1803 |
+
"method": "learned",
|
| 1804 |
+
"metric": 0.21788898058414213,
|
| 1805 |
+
"calls": 0.1533288948069241,
|
| 1806 |
+
"gflops": 14992.854896873574,
|
| 1807 |
+
"ms": 1577.6387314410913
|
| 1808 |
+
},
|
| 1809 |
+
{
|
| 1810 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1811 |
+
"method": "learned",
|
| 1812 |
+
"metric": 0.2206453715306817,
|
| 1813 |
+
"calls": 0.16624500665778957,
|
| 1814 |
+
"gflops": 16244.09951971101,
|
| 1815 |
+
"ms": 1704.5069284652548
|
| 1816 |
+
},
|
| 1817 |
+
{
|
| 1818 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1819 |
+
"method": "learned",
|
| 1820 |
+
"metric": 0.22256047317491415,
|
| 1821 |
+
"calls": 0.17569906790945405,
|
| 1822 |
+
"gflops": 17159.958985911613,
|
| 1823 |
+
"ms": 1797.3692170087152
|
| 1824 |
+
},
|
| 1825 |
+
{
|
| 1826 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1827 |
+
"method": "learned",
|
| 1828 |
+
"metric": 0.22392279063908965,
|
| 1829 |
+
"calls": 0.18202396804260984,
|
| 1830 |
+
"gflops": 17772.681868228916,
|
| 1831 |
+
"ms": 1859.495395963847
|
| 1832 |
+
},
|
| 1833 |
+
{
|
| 1834 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1835 |
+
"method": "learned",
|
| 1836 |
+
"metric": 0.22639256671969005,
|
| 1837 |
+
"calls": 0.1914780292942743,
|
| 1838 |
+
"gflops": 18688.541334429516,
|
| 1839 |
+
"ms": 1952.3576845073073
|
| 1840 |
+
},
|
| 1841 |
+
{
|
| 1842 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1843 |
+
"method": "learned",
|
| 1844 |
+
"metric": 0.2303544739053073,
|
| 1845 |
+
"calls": 0.2051264980026631,
|
| 1846 |
+
"gflops": 20010.73281732475,
|
| 1847 |
+
"ms": 2086.419439094697
|
| 1848 |
+
},
|
| 1849 |
+
{
|
| 1850 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1851 |
+
"method": "learned",
|
| 1852 |
+
"metric": 0.2328940374950383,
|
| 1853 |
+
"calls": 0.2138482023968043,
|
| 1854 |
+
"gflops": 20855.645423467562,
|
| 1855 |
+
"ms": 2172.088170074932
|
| 1856 |
+
},
|
| 1857 |
+
{
|
| 1858 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1859 |
+
"method": "learned",
|
| 1860 |
+
"metric": 0.23741897056590364,
|
| 1861 |
+
"calls": 0.22656458055925435,
|
| 1862 |
+
"gflops": 22087.540902652876,
|
| 1863 |
+
"ms": 2296.9944877636703
|
| 1864 |
+
},
|
| 1865 |
+
{
|
| 1866 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1867 |
+
"method": "learned",
|
| 1868 |
+
"metric": 0.24026689104972293,
|
| 1869 |
+
"calls": 0.23794940079893476,
|
| 1870 |
+
"gflops": 23190.44209082402,
|
| 1871 |
+
"ms": 2408.8216098829075
|
| 1872 |
+
},
|
| 1873 |
+
{
|
| 1874 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1875 |
+
"method": "learned",
|
| 1876 |
+
"metric": 0.2431042580703525,
|
| 1877 |
+
"calls": 0.2474034620505992,
|
| 1878 |
+
"gflops": 24106.30155702462,
|
| 1879 |
+
"ms": 2501.6838984263677
|
| 1880 |
+
},
|
| 1881 |
+
{
|
| 1882 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1883 |
+
"method": "learned",
|
| 1884 |
+
"metric": 0.2446393126551028,
|
| 1885 |
+
"calls": 0.2549267643142477,
|
| 1886 |
+
"gflops": 24835.11930125468,
|
| 1887 |
+
"ms": 2575.581353394051
|
| 1888 |
+
},
|
| 1889 |
+
{
|
| 1890 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1891 |
+
"method": "learned",
|
| 1892 |
+
"metric": 0.24855675020919854,
|
| 1893 |
+
"calls": 0.26198402130492676,
|
| 1894 |
+
"gflops": 25518.78904362977,
|
| 1895 |
+
"ms": 2644.901089912408
|
| 1896 |
+
},
|
| 1897 |
+
{
|
| 1898 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1899 |
+
"method": "learned",
|
| 1900 |
+
"metric": 0.25234723459722813,
|
| 1901 |
+
"calls": 0.2728362183754993,
|
| 1902 |
+
"gflops": 26570.09251539525,
|
| 1903 |
+
"ms": 2751.4965338038446
|
| 1904 |
+
},
|
| 1905 |
+
{
|
| 1906 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1907 |
+
"method": "learned",
|
| 1908 |
+
"metric": 0.2555614615399192,
|
| 1909 |
+
"calls": 0.28169107856191744,
|
| 1910 |
+
"gflops": 27427.904550639476,
|
| 1911 |
+
"ms": 2838.4731843410295
|
| 1912 |
+
},
|
| 1913 |
+
{
|
| 1914 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1915 |
+
"method": "learned",
|
| 1916 |
+
"metric": 0.25818509905301673,
|
| 1917 |
+
"calls": 0.2905459387483355,
|
| 1918 |
+
"gflops": 28285.716585883696,
|
| 1919 |
+
"ms": 2925.4498348782136
|
| 1920 |
+
},
|
| 1921 |
+
{
|
| 1922 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1923 |
+
"method": "learned",
|
| 1924 |
+
"metric": 0.2606012166289425,
|
| 1925 |
+
"calls": 0.30099866844207723,
|
| 1926 |
+
"gflops": 29298.321770344926,
|
| 1927 |
+
"ms": 3028.1215200988004
|
| 1928 |
+
},
|
| 1929 |
+
{
|
| 1930 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1931 |
+
"method": "learned",
|
| 1932 |
+
"metric": 0.2653611798288512,
|
| 1933 |
+
"calls": 0.31178428761651134,
|
| 1934 |
+
"gflops": 30343.1755275597,
|
| 1935 |
+
"ms": 3134.063004211762
|
| 1936 |
+
},
|
| 1937 |
+
{
|
| 1938 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1939 |
+
"method": "learned",
|
| 1940 |
+
"metric": 0.26884911383364996,
|
| 1941 |
+
"calls": 0.3237017310252996,
|
| 1942 |
+
"gflops": 31497.674432136508,
|
| 1943 |
+
"ms": 3251.1218045588
|
| 1944 |
+
},
|
| 1945 |
+
{
|
| 1946 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1947 |
+
"method": "learned",
|
| 1948 |
+
"metric": 0.2729730863348169,
|
| 1949 |
+
"calls": 0.336884154460719,
|
| 1950 |
+
"gflops": 32774.717913176784,
|
| 1951 |
+
"ms": 3380.6058406968637
|
| 1952 |
+
},
|
| 1953 |
+
{
|
| 1954 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1955 |
+
"method": "learned",
|
| 1956 |
+
"metric": 0.27765278557626005,
|
| 1957 |
+
"calls": 0.34800266311584555,
|
| 1958 |
+
"gflops": 33851.8202431451,
|
| 1959 |
+
"ms": 3489.817123702201
|
| 1960 |
+
},
|
| 1961 |
+
{
|
| 1962 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1963 |
+
"method": "learned",
|
| 1964 |
+
"metric": 0.2796074774091736,
|
| 1965 |
+
"calls": 0.3575233022636485,
|
| 1966 |
+
"gflops": 34774.129423896404,
|
| 1967 |
+
"ms": 3583.333372024136
|
| 1968 |
+
},
|
| 1969 |
+
{
|
| 1970 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1971 |
+
"method": "learned",
|
| 1972 |
+
"metric": 0.28204715333825625,
|
| 1973 |
+
"calls": 0.3648468708388815,
|
| 1974 |
+
"gflops": 35483.59802447434,
|
| 1975 |
+
"ms": 3655.268947656394
|
| 1976 |
+
},
|
| 1977 |
+
{
|
| 1978 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1979 |
+
"method": "learned",
|
| 1980 |
+
"metric": 0.2865153302629788,
|
| 1981 |
+
"calls": 0.37490013315579224,
|
| 1982 |
+
"gflops": 36457.504921631305,
|
| 1983 |
+
"ms": 3754.0168742061296
|
| 1984 |
+
},
|
| 1985 |
+
{
|
| 1986 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1987 |
+
"method": "learned",
|
| 1988 |
+
"metric": 0.2895286620478625,
|
| 1989 |
+
"calls": 0.3859520639147802,
|
| 1990 |
+
"gflops": 37528.15753704891,
|
| 1991 |
+
"ms": 3862.574197432991
|
| 1992 |
+
},
|
| 1993 |
+
{
|
| 1994 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 1995 |
+
"method": "learned",
|
| 1996 |
+
"metric": 0.2924523439388038,
|
| 1997 |
+
"calls": 0.39387483355525965,
|
| 1998 |
+
"gflops": 38295.673568583225,
|
| 1999 |
+
"ms": 3940.3954110715254
|
| 2000 |
+
},
|
| 2001 |
+
{
|
| 2002 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2003 |
+
"method": "learned",
|
| 2004 |
+
"metric": 0.29493220967499056,
|
| 2005 |
+
"calls": 0.403595206391478,
|
| 2006 |
+
"gflops": 39237.33189298665,
|
| 2007 |
+
"ms": 4035.8735387288853
|
| 2008 |
+
},
|
| 2009 |
+
{
|
| 2010 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2011 |
+
"method": "learned",
|
| 2012 |
+
"metric": 0.29679244853296216,
|
| 2013 |
+
"calls": 0.4097869507323568,
|
| 2014 |
+
"gflops": 39837.15534620254,
|
| 2015 |
+
"ms": 4096.691798127067
|
| 2016 |
+
},
|
| 2017 |
+
{
|
| 2018 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2019 |
+
"method": "learned",
|
| 2020 |
+
"metric": 0.29824032060818284,
|
| 2021 |
+
"calls": 0.4171105193075899,
|
| 2022 |
+
"gflops": 40546.62394678048,
|
| 2023 |
+
"ms": 4168.627373759326
|
| 2024 |
+
},
|
| 2025 |
+
{
|
| 2026 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2027 |
+
"method": "learned",
|
| 2028 |
+
"metric": 0.30060258544213647,
|
| 2029 |
+
"calls": 0.42676431424766975,
|
| 2030 |
+
"gflops": 41481.8325566332,
|
| 2031 |
+
"ms": 4263.45154163821
|
| 2032 |
+
},
|
| 2033 |
+
{
|
| 2034 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2035 |
+
"method": "learned",
|
| 2036 |
+
"metric": 0.30092311829652235,
|
| 2037 |
+
"calls": 0.4334886817576565,
|
| 2038 |
+
"gflops": 42133.253726254756,
|
| 2039 |
+
"ms": 4329.501479264193
|
| 2040 |
+
},
|
| 2041 |
+
{
|
| 2042 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2043 |
+
"method": "learned",
|
| 2044 |
+
"metric": 0.29908337970039717,
|
| 2045 |
+
"calls": 0.44560585885486015,
|
| 2046 |
+
"gflops": 43307.101774483686,
|
| 2047 |
+
"ms": 4448.5221589466555
|
| 2048 |
+
},
|
| 2049 |
+
{
|
| 2050 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2051 |
+
"method": "learned",
|
| 2052 |
+
"metric": 0.2977899223131896,
|
| 2053 |
+
"calls": 0.4533288948069242,
|
| 2054 |
+
"gflops": 44055.26866236588,
|
| 2055 |
+
"ms": 4524.381493249765
|
| 2056 |
+
},
|
| 2057 |
+
{
|
| 2058 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2059 |
+
"method": "learned",
|
| 2060 |
+
"metric": 0.2982768381293317,
|
| 2061 |
+
"calls": 0.45952063914780295,
|
| 2062 |
+
"gflops": 44655.09211558176,
|
| 2063 |
+
"ms": 4585.199752647946
|
| 2064 |
+
},
|
| 2065 |
+
{
|
| 2066 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2067 |
+
"method": "learned",
|
| 2068 |
+
"metric": 0.29713930300206126,
|
| 2069 |
+
"calls": 0.46677762982689747,
|
| 2070 |
+
"gflops": 45358.11100160898,
|
| 2071 |
+
"ms": 4656.481368501729
|
| 2072 |
+
},
|
| 2073 |
+
{
|
| 2074 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2075 |
+
"method": "learned",
|
| 2076 |
+
"metric": 0.2981419006299502,
|
| 2077 |
+
"calls": 0.4787616511318243,
|
| 2078 |
+
"gflops": 46519.05962073651,
|
| 2079 |
+
"ms": 4774.194128627242
|
| 2080 |
+
},
|
| 2081 |
+
{
|
| 2082 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2083 |
+
"method": "learned",
|
| 2084 |
+
"metric": 0.30134517763453006,
|
| 2085 |
+
"calls": 0.4900133155792277,
|
| 2086 |
+
"gflops": 47609.06137980623,
|
| 2087 |
+
"ms": 4884.7133311895295
|
| 2088 |
+
},
|
| 2089 |
+
{
|
| 2090 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2091 |
+
"method": "learned",
|
| 2092 |
+
"metric": 0.3013386133731364,
|
| 2093 |
+
"calls": 0.5019973368841545,
|
| 2094 |
+
"gflops": 48770.009998933754,
|
| 2095 |
+
"ms": 5002.426091315042
|
| 2096 |
+
},
|
| 2097 |
+
{
|
| 2098 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2099 |
+
"method": "learned",
|
| 2100 |
+
"metric": 0.3006032949818893,
|
| 2101 |
+
"calls": 0.5176431424766977,
|
| 2102 |
+
"gflops": 50285.69291835024,
|
| 2103 |
+
"ms": 5156.106639256684
|
| 2104 |
+
},
|
| 2105 |
+
{
|
| 2106 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2107 |
+
"method": "learned",
|
| 2108 |
+
"metric": 0.29926650757486717,
|
| 2109 |
+
"calls": 0.5346205059920106,
|
| 2110 |
+
"gflops": 51930.370128780894,
|
| 2111 |
+
"ms": 5322.8663827678265
|
| 2112 |
+
},
|
| 2113 |
+
{
|
| 2114 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2115 |
+
"method": "learned",
|
| 2116 |
+
"metric": 0.29911586063303675,
|
| 2117 |
+
"calls": 0.5474700399467377,
|
| 2118 |
+
"gflops": 53175.16503706763,
|
| 2119 |
+
"ms": 5449.080620013516
|
| 2120 |
+
},
|
| 2121 |
+
{
|
| 2122 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2123 |
+
"method": "learned",
|
| 2124 |
+
"metric": 0.29813893282180814,
|
| 2125 |
+
"calls": 0.5575898801597869,
|
| 2126 |
+
"gflops": 54155.52164877531,
|
| 2127 |
+
"ms": 5548.482506341727
|
| 2128 |
+
},
|
| 2129 |
+
{
|
| 2130 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2131 |
+
"method": "learned",
|
| 2132 |
+
"metric": 0.2974509950363013,
|
| 2133 |
+
"calls": 0.5675765645805592,
|
| 2134 |
+
"gflops": 55122.97883138157,
|
| 2135 |
+
"ms": 5646.576473112987
|
| 2136 |
+
},
|
| 2137 |
+
{
|
| 2138 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2139 |
+
"method": "learned",
|
| 2140 |
+
"metric": 0.29724357969129245,
|
| 2141 |
+
"calls": 0.5804260985352863,
|
| 2142 |
+
"gflops": 56367.773739668315,
|
| 2143 |
+
"ms": 5772.790710358676
|
| 2144 |
+
},
|
| 2145 |
+
{
|
| 2146 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2147 |
+
"method": "learned",
|
| 2148 |
+
"metric": 0.29665775151403084,
|
| 2149 |
+
"calls": 0.5938082556591212,
|
| 2150 |
+
"gflops": 57664.16636436072,
|
| 2151 |
+
"ms": 5904.236625832166
|
| 2152 |
+
},
|
| 2153 |
+
{
|
| 2154 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2155 |
+
"method": "learned",
|
| 2156 |
+
"metric": 0.2975661409457673,
|
| 2157 |
+
"calls": 0.603595206391478,
|
| 2158 |
+
"gflops": 58612.274403314856,
|
| 2159 |
+
"ms": 6000.368713268001
|
| 2160 |
+
},
|
| 2161 |
+
{
|
| 2162 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2163 |
+
"method": "learned",
|
| 2164 |
+
"metric": 0.29631335243812956,
|
| 2165 |
+
"calls": 0.6135153129161118,
|
| 2166 |
+
"gflops": 59573.28187137042,
|
| 2167 |
+
"ms": 6097.808720260787
|
| 2168 |
+
},
|
| 2169 |
+
{
|
| 2170 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2171 |
+
"method": "learned",
|
| 2172 |
+
"metric": 0.2960348088956203,
|
| 2173 |
+
"calls": 0.6214380825565912,
|
| 2174 |
+
"gflops": 60340.79790290472,
|
| 2175 |
+
"ms": 6175.62993389932
|
| 2176 |
+
},
|
| 2177 |
+
{
|
| 2178 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2179 |
+
"method": "learned",
|
| 2180 |
+
"metric": 0.2951989613231927,
|
| 2181 |
+
"calls": 0.6318242343541943,
|
| 2182 |
+
"gflops": 61346.95337281523,
|
| 2183 |
+
"ms": 6277.647659341431
|
| 2184 |
+
},
|
| 2185 |
+
{
|
| 2186 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2187 |
+
"method": "learned",
|
| 2188 |
+
"metric": 0.29386485972911613,
|
| 2189 |
+
"calls": 0.6431424766977363,
|
| 2190 |
+
"gflops": 62443.404846435675,
|
| 2191 |
+
"ms": 6388.820821682193
|
| 2192 |
+
},
|
| 2193 |
+
{
|
| 2194 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2195 |
+
"method": "learned",
|
| 2196 |
+
"metric": 0.292479005789999,
|
| 2197 |
+
"calls": 0.6534620505992011,
|
| 2198 |
+
"gflops": 63443.110601795495,
|
| 2199 |
+
"ms": 6490.184587345831
|
| 2200 |
+
},
|
| 2201 |
+
{
|
| 2202 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2203 |
+
"method": "learned",
|
| 2204 |
+
"metric": 0.2926539927366717,
|
| 2205 |
+
"calls": 0.6655126498002664,
|
| 2206 |
+
"gflops": 64610.50893547373,
|
| 2207 |
+
"ms": 6608.551307249818
|
| 2208 |
+
},
|
| 2209 |
+
{
|
| 2210 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2211 |
+
"method": "learned",
|
| 2212 |
+
"metric": 0.29278795045342987,
|
| 2213 |
+
"calls": 0.6742343541944075,
|
| 2214 |
+
"gflops": 65455.42154161653,
|
| 2215 |
+
"ms": 6694.2200382300525
|
| 2216 |
+
},
|
| 2217 |
+
{
|
| 2218 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2219 |
+
"method": "learned",
|
| 2220 |
+
"metric": 0.29144368638516527,
|
| 2221 |
+
"calls": 0.6844207723035952,
|
| 2222 |
+
"gflops": 66442.22786787493,
|
| 2223 |
+
"ms": 6794.275884336738
|
| 2224 |
+
},
|
| 2225 |
+
{
|
| 2226 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2227 |
+
"method": "learned",
|
| 2228 |
+
"metric": 0.2913839507485529,
|
| 2229 |
+
"calls": 0.6950732356857523,
|
| 2230 |
+
"gflops": 67474.18219598828,
|
| 2231 |
+
"ms": 6898.909448892749
|
| 2232 |
+
},
|
| 2233 |
+
{
|
| 2234 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2235 |
+
"method": "learned",
|
| 2236 |
+
"metric": 0.2916685120367808,
|
| 2237 |
+
"calls": 0.70219707057257,
|
| 2238 |
+
"gflops": 68164.30165291409,
|
| 2239 |
+
"ms": 6968.883145189583
|
| 2240 |
+
},
|
| 2241 |
+
{
|
| 2242 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2243 |
+
"method": "learned",
|
| 2244 |
+
"metric": 0.2908903102338221,
|
| 2245 |
+
"calls": 0.70745672436751,
|
| 2246 |
+
"gflops": 68673.82910242004,
|
| 2247 |
+
"ms": 7020.545967689112
|
| 2248 |
+
},
|
| 2249 |
+
{
|
| 2250 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2251 |
+
"method": "learned",
|
| 2252 |
+
"metric": 0.29068952549932253,
|
| 2253 |
+
"calls": 0.7203728362183754,
|
| 2254 |
+
"gflops": 69925.07372525749,
|
| 2255 |
+
"ms": 7147.4141647132765
|
| 2256 |
+
},
|
| 2257 |
+
{
|
| 2258 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2259 |
+
"method": "learned",
|
| 2260 |
+
"metric": 0.28982721462710204,
|
| 2261 |
+
"calls": 0.7292942743009322,
|
| 2262 |
+
"gflops": 70789.33547505244,
|
| 2263 |
+
"ms": 7235.044775028938
|
| 2264 |
+
},
|
| 2265 |
+
{
|
| 2266 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2267 |
+
"method": "learned",
|
| 2268 |
+
"metric": 0.289887284031958,
|
| 2269 |
+
"calls": 0.7372836218375498,
|
| 2270 |
+
"gflops": 71563.30122113743,
|
| 2271 |
+
"ms": 7313.519948445944
|
| 2272 |
+
},
|
| 2273 |
+
{
|
| 2274 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2275 |
+
"method": "learned",
|
| 2276 |
+
"metric": 0.2914457139433469,
|
| 2277 |
+
"calls": 0.7460719041278296,
|
| 2278 |
+
"gflops": 72414.66354183097,
|
| 2279 |
+
"ms": 7399.842639204656
|
| 2280 |
+
},
|
| 2281 |
+
{
|
| 2282 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2283 |
+
"method": "learned",
|
| 2284 |
+
"metric": 0.29113442803741185,
|
| 2285 |
+
"calls": 0.7552596537949402,
|
| 2286 |
+
"gflops": 73304.72414982873,
|
| 2287 |
+
"ms": 7490.089088634216
|
| 2288 |
+
},
|
| 2289 |
+
{
|
| 2290 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2291 |
+
"method": "learned",
|
| 2292 |
+
"metric": 0.29031315276962183,
|
| 2293 |
+
"calls": 0.7674434087882822,
|
| 2294 |
+
"gflops": 74485.02191260837,
|
| 2295 |
+
"ms": 7609.763728095152
|
| 2296 |
+
},
|
| 2297 |
+
{
|
| 2298 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2299 |
+
"method": "learned",
|
| 2300 |
+
"metric": 0.289990968297447,
|
| 2301 |
+
"calls": 0.7755659121171771,
|
| 2302 |
+
"gflops": 75271.8870877948,
|
| 2303 |
+
"ms": 7689.5468210691115
|
| 2304 |
+
},
|
| 2305 |
+
{
|
| 2306 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2307 |
+
"method": "learned",
|
| 2308 |
+
"metric": 0.2901920651624007,
|
| 2309 |
+
"calls": 0.7833555259653795,
|
| 2310 |
+
"gflops": 76026.50369022769,
|
| 2311 |
+
"ms": 7766.060115150694
|
| 2312 |
+
},
|
| 2313 |
+
{
|
| 2314 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2315 |
+
"method": "learned",
|
| 2316 |
+
"metric": 0.2901181158873772,
|
| 2317 |
+
"calls": 0.7904793608521972,
|
| 2318 |
+
"gflops": 76716.62314715351,
|
| 2319 |
+
"ms": 7836.033811447528
|
| 2320 |
+
},
|
| 2321 |
+
{
|
| 2322 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2323 |
+
"method": "learned",
|
| 2324 |
+
"metric": 0.28973901353907683,
|
| 2325 |
+
"calls": 0.7987350199733687,
|
| 2326 |
+
"gflops": 77516.38775144133,
|
| 2327 |
+
"ms": 7917.124823978435
|
| 2328 |
+
},
|
| 2329 |
+
{
|
| 2330 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2331 |
+
"method": "learned",
|
| 2332 |
+
"metric": 0.28888237446044834,
|
| 2333 |
+
"calls": 0.8127829560585885,
|
| 2334 |
+
"gflops": 78877.27752164083,
|
| 2335 |
+
"ms": 8055.110337236676
|
| 2336 |
+
},
|
| 2337 |
+
{
|
| 2338 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2339 |
+
"method": "learned",
|
| 2340 |
+
"metric": 0.2890360783246395,
|
| 2341 |
+
"calls": 0.8225699067909453,
|
| 2342 |
+
"gflops": 79825.38556059496,
|
| 2343 |
+
"ms": 8151.242424672511
|
| 2344 |
+
},
|
| 2345 |
+
{
|
| 2346 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2347 |
+
"method": "learned",
|
| 2348 |
+
"metric": 0.28767900194586976,
|
| 2349 |
+
"calls": 0.8346205059920105,
|
| 2350 |
+
"gflops": 80992.78389427319,
|
| 2351 |
+
"ms": 8269.6091445765
|
| 2352 |
+
},
|
| 2353 |
+
{
|
| 2354 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2355 |
+
"method": "learned",
|
| 2356 |
+
"metric": 0.28869706841582,
|
| 2357 |
+
"calls": 0.8463382157123837,
|
| 2358 |
+
"gflops": 82127.9336551979,
|
| 2359 |
+
"ms": 8384.706065588114
|
| 2360 |
+
},
|
| 2361 |
+
{
|
| 2362 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2363 |
+
"method": "learned",
|
| 2364 |
+
"metric": 0.2875418385011496,
|
| 2365 |
+
"calls": 0.8567243675099867,
|
| 2366 |
+
"gflops": 83134.08912510841,
|
| 2367 |
+
"ms": 8486.723791030225
|
| 2368 |
+
},
|
| 2369 |
+
{
|
| 2370 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2371 |
+
"method": "learned",
|
| 2372 |
+
"metric": 0.28776751513017684,
|
| 2373 |
+
"calls": 0.86491344873502,
|
| 2374 |
+
"gflops": 83927.40401484555,
|
| 2375 |
+
"ms": 8567.160843782658
|
| 2376 |
+
},
|
| 2377 |
+
{
|
| 2378 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2379 |
+
"method": "learned",
|
| 2380 |
+
"metric": 0.2880879394318149,
|
| 2381 |
+
"calls": 0.8767643142476697,
|
| 2382 |
+
"gflops": 85075.45320487165,
|
| 2383 |
+
"ms": 8683.565684351219
|
| 2384 |
+
},
|
| 2385 |
+
{
|
| 2386 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2387 |
+
"method": "learned",
|
| 2388 |
+
"metric": 0.287273877029594,
|
| 2389 |
+
"calls": 0.8856857523302264,
|
| 2390 |
+
"gflops": 85939.71495466659,
|
| 2391 |
+
"ms": 8771.196294666881
|
| 2392 |
+
},
|
| 2393 |
+
{
|
| 2394 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2395 |
+
"method": "learned",
|
| 2396 |
+
"metric": 0.28547632945289336,
|
| 2397 |
+
"calls": 0.8951398135818909,
|
| 2398 |
+
"gflops": 86855.5744208672,
|
| 2399 |
+
"ms": 8864.05858321034
|
| 2400 |
+
},
|
| 2401 |
+
{
|
| 2402 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2403 |
+
"method": "learned",
|
| 2404 |
+
"metric": 0.284269379519687,
|
| 2405 |
+
"calls": 0.9029960053262316,
|
| 2406 |
+
"gflops": 87616.64073785077,
|
| 2407 |
+
"ms": 8941.225837070398
|
| 2408 |
+
},
|
| 2409 |
+
{
|
| 2410 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2411 |
+
"method": "learned",
|
| 2412 |
+
"metric": 0.28250852235623786,
|
| 2413 |
+
"calls": 0.9131824234354193,
|
| 2414 |
+
"gflops": 88603.44706410918,
|
| 2415 |
+
"ms": 9041.281683177083
|
| 2416 |
+
},
|
| 2417 |
+
{
|
| 2418 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2419 |
+
"method": "learned",
|
| 2420 |
+
"metric": 0.2814829677111451,
|
| 2421 |
+
"calls": 0.9243675099866845,
|
| 2422 |
+
"gflops": 89686.99910862821,
|
| 2423 |
+
"ms": 9151.146925960897
|
| 2424 |
+
},
|
| 2425 |
+
{
|
| 2426 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2427 |
+
"method": "learned",
|
| 2428 |
+
"metric": 0.2800886018148173,
|
| 2429 |
+
"calls": 0.9328229027962717,
|
| 2430 |
+
"gflops": 90506.11285656819,
|
| 2431 |
+
"ms": 9234.199817827232
|
| 2432 |
+
},
|
| 2433 |
+
{
|
| 2434 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2435 |
+
"method": "learned",
|
| 2436 |
+
"metric": 0.27854520316604037,
|
| 2437 |
+
"calls": 0.9400798934753661,
|
| 2438 |
+
"gflops": 91209.13174259539,
|
| 2439 |
+
"ms": 9305.481433681012
|
| 2440 |
+
},
|
| 2441 |
+
{
|
| 2442 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2443 |
+
"method": "learned",
|
| 2444 |
+
"metric": 0.27746458817138653,
|
| 2445 |
+
"calls": 0.9482689747003995,
|
| 2446 |
+
"gflops": 92002.44663233255,
|
| 2447 |
+
"ms": 9385.918486433447
|
| 2448 |
+
},
|
| 2449 |
+
{
|
| 2450 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2451 |
+
"method": "learned",
|
| 2452 |
+
"metric": 0.2760878894837333,
|
| 2453 |
+
"calls": 0.9603861517976032,
|
| 2454 |
+
"gflops": 93176.29468056148,
|
| 2455 |
+
"ms": 9504.939166115912
|
| 2456 |
+
},
|
| 2457 |
+
{
|
| 2458 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2459 |
+
"method": "learned",
|
| 2460 |
+
"metric": 0.27468746115763226,
|
| 2461 |
+
"calls": 0.9677762982689748,
|
| 2462 |
+
"gflops": 93892.21299569013,
|
| 2463 |
+
"ms": 9577.528701526644
|
| 2464 |
+
},
|
| 2465 |
+
{
|
| 2466 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2467 |
+
"method": "learned",
|
| 2468 |
+
"metric": 0.2732948963509506,
|
| 2469 |
+
"calls": 0.9782290279627164,
|
| 2470 |
+
"gflops": 94904.81818015134,
|
| 2471 |
+
"ms": 9680.20038674723
|
| 2472 |
+
},
|
| 2473 |
+
{
|
| 2474 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2475 |
+
"method": "learned",
|
| 2476 |
+
"metric": 0.27348968306795485,
|
| 2477 |
+
"calls": 0.9849533954727031,
|
| 2478 |
+
"gflops": 95556.2393497729,
|
| 2479 |
+
"ms": 9746.250324373212
|
| 2480 |
+
},
|
| 2481 |
+
{
|
| 2482 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2483 |
+
"method": "learned",
|
| 2484 |
+
"metric": 0.27001009067886245,
|
| 2485 |
+
"calls": 1.0,
|
| 2486 |
+
"gflops": 97013.87483823301,
|
| 2487 |
+
"ms": 9894.045234308578
|
| 2488 |
+
},
|
| 2489 |
+
{
|
| 2490 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2491 |
+
"method": "confidence",
|
| 2492 |
+
"metric": 0.18129624718318377,
|
| 2493 |
+
"calls": 0.0,
|
| 2494 |
+
"gflops": 135.97853184000002,
|
| 2495 |
+
"ms": 46.30809425361804
|
| 2496 |
+
},
|
| 2497 |
+
{
|
| 2498 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2499 |
+
"method": "confidence",
|
| 2500 |
+
"metric": 0.18129624718318377,
|
| 2501 |
+
"calls": 0.0,
|
| 2502 |
+
"gflops": 135.97853184000002,
|
| 2503 |
+
"ms": 46.30809425361804
|
| 2504 |
+
},
|
| 2505 |
+
{
|
| 2506 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2507 |
+
"method": "confidence",
|
| 2508 |
+
"metric": 0.18129624718318377,
|
| 2509 |
+
"calls": 0.0,
|
| 2510 |
+
"gflops": 135.97853184000002,
|
| 2511 |
+
"ms": 46.30809425361804
|
| 2512 |
+
},
|
| 2513 |
+
{
|
| 2514 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2515 |
+
"method": "confidence",
|
| 2516 |
+
"metric": 0.18129624718318377,
|
| 2517 |
+
"calls": 0.0,
|
| 2518 |
+
"gflops": 135.97853184000002,
|
| 2519 |
+
"ms": 46.30809425361804
|
| 2520 |
+
},
|
| 2521 |
+
{
|
| 2522 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2523 |
+
"method": "confidence",
|
| 2524 |
+
"metric": 0.18129624718318377,
|
| 2525 |
+
"calls": 0.0,
|
| 2526 |
+
"gflops": 135.97853184000002,
|
| 2527 |
+
"ms": 46.30809425361804
|
| 2528 |
+
},
|
| 2529 |
+
{
|
| 2530 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2531 |
+
"method": "confidence",
|
| 2532 |
+
"metric": 0.18129624718318377,
|
| 2533 |
+
"calls": 0.0,
|
| 2534 |
+
"gflops": 135.97853184000002,
|
| 2535 |
+
"ms": 46.30809425361804
|
| 2536 |
+
},
|
| 2537 |
+
{
|
| 2538 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2539 |
+
"method": "confidence",
|
| 2540 |
+
"metric": 0.18129624718318377,
|
| 2541 |
+
"calls": 0.0,
|
| 2542 |
+
"gflops": 135.97853184000002,
|
| 2543 |
+
"ms": 46.30809425361804
|
| 2544 |
+
},
|
| 2545 |
+
{
|
| 2546 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2547 |
+
"method": "confidence",
|
| 2548 |
+
"metric": 0.18129624718318377,
|
| 2549 |
+
"calls": 0.0,
|
| 2550 |
+
"gflops": 135.97853184000002,
|
| 2551 |
+
"ms": 46.30809425361804
|
| 2552 |
+
},
|
| 2553 |
+
{
|
| 2554 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2555 |
+
"method": "confidence",
|
| 2556 |
+
"metric": 0.18129624718318377,
|
| 2557 |
+
"calls": 0.0,
|
| 2558 |
+
"gflops": 135.97853184000002,
|
| 2559 |
+
"ms": 46.30809425361804
|
| 2560 |
+
},
|
| 2561 |
+
{
|
| 2562 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2563 |
+
"method": "confidence",
|
| 2564 |
+
"metric": 0.18129624718318377,
|
| 2565 |
+
"calls": 0.0,
|
| 2566 |
+
"gflops": 135.97853184000002,
|
| 2567 |
+
"ms": 46.30809425361804
|
| 2568 |
+
},
|
| 2569 |
+
{
|
| 2570 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2571 |
+
"method": "confidence",
|
| 2572 |
+
"metric": 0.18129624718318377,
|
| 2573 |
+
"calls": 0.0,
|
| 2574 |
+
"gflops": 135.97853184000002,
|
| 2575 |
+
"ms": 46.30809425361804
|
| 2576 |
+
},
|
| 2577 |
+
{
|
| 2578 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2579 |
+
"method": "confidence",
|
| 2580 |
+
"metric": 0.18129624718318377,
|
| 2581 |
+
"calls": 0.0,
|
| 2582 |
+
"gflops": 135.97853184000002,
|
| 2583 |
+
"ms": 46.30809425361804
|
| 2584 |
+
},
|
| 2585 |
+
{
|
| 2586 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2587 |
+
"method": "confidence",
|
| 2588 |
+
"metric": 0.18129624718318377,
|
| 2589 |
+
"calls": 0.0,
|
| 2590 |
+
"gflops": 135.97853184000002,
|
| 2591 |
+
"ms": 46.30809425361804
|
| 2592 |
+
},
|
| 2593 |
+
{
|
| 2594 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2595 |
+
"method": "confidence",
|
| 2596 |
+
"metric": 0.18129624718318377,
|
| 2597 |
+
"calls": 0.0,
|
| 2598 |
+
"gflops": 135.97853184000002,
|
| 2599 |
+
"ms": 46.30809425361804
|
| 2600 |
+
},
|
| 2601 |
+
{
|
| 2602 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2603 |
+
"method": "confidence",
|
| 2604 |
+
"metric": 0.18129624718318377,
|
| 2605 |
+
"calls": 0.0,
|
| 2606 |
+
"gflops": 135.97853184000002,
|
| 2607 |
+
"ms": 46.30809425361804
|
| 2608 |
+
},
|
| 2609 |
+
{
|
| 2610 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2611 |
+
"method": "confidence",
|
| 2612 |
+
"metric": 0.18129624718318377,
|
| 2613 |
+
"calls": 0.0,
|
| 2614 |
+
"gflops": 135.97853184000002,
|
| 2615 |
+
"ms": 46.30809425361804
|
| 2616 |
+
},
|
| 2617 |
+
{
|
| 2618 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2619 |
+
"method": "confidence",
|
| 2620 |
+
"metric": 0.18129624718318377,
|
| 2621 |
+
"calls": 0.0,
|
| 2622 |
+
"gflops": 135.97853184000002,
|
| 2623 |
+
"ms": 46.30809425361804
|
| 2624 |
+
},
|
| 2625 |
+
{
|
| 2626 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2627 |
+
"method": "confidence",
|
| 2628 |
+
"metric": 0.18129624718318377,
|
| 2629 |
+
"calls": 0.0,
|
| 2630 |
+
"gflops": 135.97853184000002,
|
| 2631 |
+
"ms": 46.30809425361804
|
| 2632 |
+
},
|
| 2633 |
+
{
|
| 2634 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2635 |
+
"method": "confidence",
|
| 2636 |
+
"metric": 0.18129624718318377,
|
| 2637 |
+
"calls": 0.0,
|
| 2638 |
+
"gflops": 135.97853184000002,
|
| 2639 |
+
"ms": 46.30809425361804
|
| 2640 |
+
},
|
| 2641 |
+
{
|
| 2642 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2643 |
+
"method": "confidence",
|
| 2644 |
+
"metric": 0.1784481207469488,
|
| 2645 |
+
"calls": 0.17942743009320905,
|
| 2646 |
+
"gflops": 17517.959245999285,
|
| 2647 |
+
"ms": 1808.7296972439362
|
| 2648 |
+
},
|
| 2649 |
+
{
|
| 2650 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2651 |
+
"method": "confidence",
|
| 2652 |
+
"metric": 0.1780928651788194,
|
| 2653 |
+
"calls": 0.19340878828229027,
|
| 2654 |
+
"gflops": 18872.39930164806,
|
| 2655 |
+
"ms": 1946.0612507237013
|
| 2656 |
+
},
|
| 2657 |
+
{
|
| 2658 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2659 |
+
"method": "confidence",
|
| 2660 |
+
"metric": 0.17829989216687975,
|
| 2661 |
+
"calls": 0.20306258322237017,
|
| 2662 |
+
"gflops": 19807.60791150079,
|
| 2663 |
+
"ms": 2040.8854186025867
|
| 2664 |
+
},
|
| 2665 |
+
{
|
| 2666 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2667 |
+
"method": "confidence",
|
| 2668 |
+
"metric": 0.1779854683890517,
|
| 2669 |
+
"calls": 0.20838881491344874,
|
| 2670 |
+
"gflops": 20323.585075557465,
|
| 2671 |
+
"ms": 2093.2022008805925
|
| 2672 |
+
},
|
| 2673 |
+
{
|
| 2674 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2675 |
+
"method": "confidence",
|
| 2676 |
+
"metric": 0.17917657728013692,
|
| 2677 |
+
"calls": 0.21537949400798934,
|
| 2678 |
+
"gflops": 21000.805103381852,
|
| 2679 |
+
"ms": 2161.867977620475
|
| 2680 |
+
},
|
| 2681 |
+
{
|
| 2682 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2683 |
+
"method": "confidence",
|
| 2684 |
+
"metric": 0.17803687432776102,
|
| 2685 |
+
"calls": 0.23135818908122505,
|
| 2686 |
+
"gflops": 22548.736595551884,
|
| 2687 |
+
"ms": 2318.8183244544925
|
| 2688 |
+
},
|
| 2689 |
+
{
|
| 2690 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2691 |
+
"method": "confidence",
|
| 2692 |
+
"metric": 0.18026802885144627,
|
| 2693 |
+
"calls": 0.24800266311584554,
|
| 2694 |
+
"gflops": 24161.165233228996,
|
| 2695 |
+
"ms": 2482.3082690732604
|
| 2696 |
+
},
|
| 2697 |
+
{
|
| 2698 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2699 |
+
"method": "confidence",
|
| 2700 |
+
"metric": 0.18119827871024874,
|
| 2701 |
+
"calls": 0.2589880159786951,
|
| 2702 |
+
"gflops": 25225.368134095894,
|
| 2703 |
+
"ms": 2590.211632521647
|
| 2704 |
+
},
|
| 2705 |
+
{
|
| 2706 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2707 |
+
"method": "confidence",
|
| 2708 |
+
"metric": 0.17963665085316138,
|
| 2709 |
+
"calls": 0.27430093209054596,
|
| 2710 |
+
"gflops": 26708.802480758837,
|
| 2711 |
+
"ms": 2740.6223815709136
|
| 2712 |
+
},
|
| 2713 |
+
{
|
| 2714 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2715 |
+
"method": "confidence",
|
| 2716 |
+
"metric": 0.17924095839307458,
|
| 2717 |
+
"calls": 0.281624500665779,
|
| 2718 |
+
"gflops": 27418.271081336767,
|
| 2719 |
+
"ms": 2812.5579572031716
|
| 2720 |
+
},
|
| 2721 |
+
{
|
| 2722 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2723 |
+
"method": "confidence",
|
| 2724 |
+
"metric": 0.1789157174618718,
|
| 2725 |
+
"calls": 0.28761651131824234,
|
| 2726 |
+
"gflops": 27998.745390900527,
|
| 2727 |
+
"ms": 2871.4143372659278
|
| 2728 |
+
},
|
| 2729 |
+
{
|
| 2730 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2731 |
+
"method": "confidence",
|
| 2732 |
+
"metric": 0.1787206845800119,
|
| 2733 |
+
"calls": 0.2969374167776298,
|
| 2734 |
+
"gflops": 28901.70542799971,
|
| 2735 |
+
"ms": 2962.9687062524376
|
| 2736 |
+
},
|
| 2737 |
+
{
|
| 2738 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2739 |
+
"method": "confidence",
|
| 2740 |
+
"metric": 0.17813242189706796,
|
| 2741 |
+
"calls": 0.30459387483355527,
|
| 2742 |
+
"gflops": 29643.422601331185,
|
| 2743 |
+
"ms": 3038.174080777071
|
| 2744 |
+
},
|
| 2745 |
+
{
|
| 2746 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2747 |
+
"method": "confidence",
|
| 2748 |
+
"metric": 0.17772000842137498,
|
| 2749 |
+
"calls": 0.3129161118508655,
|
| 2750 |
+
"gflops": 30449.636920169738,
|
| 2751 |
+
"ms": 3119.919053086455
|
| 2752 |
+
},
|
| 2753 |
+
{
|
| 2754 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2755 |
+
"method": "confidence",
|
| 2756 |
+
"metric": 0.17791259734473341,
|
| 2757 |
+
"calls": 0.31990679094540614,
|
| 2758 |
+
"gflops": 31126.85694799413,
|
| 2759 |
+
"ms": 3188.5848298263377
|
| 2760 |
+
},
|
| 2761 |
+
{
|
| 2762 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2763 |
+
"method": "confidence",
|
| 2764 |
+
"metric": 0.1789653934599337,
|
| 2765 |
+
"calls": 0.3322237017310253,
|
| 2766 |
+
"gflops": 32320.05413987519,
|
| 2767 |
+
"ms": 3309.5673888442257
|
| 2768 |
+
},
|
| 2769 |
+
{
|
| 2770 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2771 |
+
"method": "confidence",
|
| 2772 |
+
"metric": 0.18042682200794885,
|
| 2773 |
+
"calls": 0.34054593874833555,
|
| 2774 |
+
"gflops": 33126.268458713756,
|
| 2775 |
+
"ms": 3391.31236115361
|
| 2776 |
+
},
|
| 2777 |
+
{
|
| 2778 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2779 |
+
"method": "confidence",
|
| 2780 |
+
"metric": 0.17962877325912951,
|
| 2781 |
+
"calls": 0.3488681757656458,
|
| 2782 |
+
"gflops": 33932.48277755231,
|
| 2783 |
+
"ms": 3473.057333462994
|
| 2784 |
+
},
|
| 2785 |
+
{
|
| 2786 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2787 |
+
"method": "confidence",
|
| 2788 |
+
"metric": 0.18158304685510065,
|
| 2789 |
+
"calls": 0.3581890812250333,
|
| 2790 |
+
"gflops": 34835.44281465149,
|
| 2791 |
+
"ms": 3564.611702449504
|
| 2792 |
+
},
|
| 2793 |
+
{
|
| 2794 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2795 |
+
"method": "confidence",
|
| 2796 |
+
"metric": 0.18069239723097547,
|
| 2797 |
+
"calls": 0.37017310252996005,
|
| 2798 |
+
"gflops": 35996.39143377901,
|
| 2799 |
+
"ms": 3682.3244625750167
|
| 2800 |
+
},
|
| 2801 |
+
{
|
| 2802 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2803 |
+
"method": "confidence",
|
| 2804 |
+
"metric": 0.1812845803851709,
|
| 2805 |
+
"calls": 0.37683089214380827,
|
| 2806 |
+
"gflops": 36641.362888849864,
|
| 2807 |
+
"ms": 3747.720440422524
|
| 2808 |
+
},
|
| 2809 |
+
{
|
| 2810 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2811 |
+
"method": "confidence",
|
| 2812 |
+
"metric": 0.1824781664689562,
|
| 2813 |
+
"calls": 0.38348868175765644,
|
| 2814 |
+
"gflops": 37286.3343439207,
|
| 2815 |
+
"ms": 3813.116418270031
|
| 2816 |
+
},
|
| 2817 |
+
{
|
| 2818 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2819 |
+
"method": "confidence",
|
| 2820 |
+
"metric": 0.18220594501399923,
|
| 2821 |
+
"calls": 0.4021304926764314,
|
| 2822 |
+
"gflops": 39092.254418119075,
|
| 2823 |
+
"ms": 3996.225156243051
|
| 2824 |
+
},
|
| 2825 |
+
{
|
| 2826 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2827 |
+
"method": "confidence",
|
| 2828 |
+
"metric": 0.18162498930772586,
|
| 2829 |
+
"calls": 0.41311584553928093,
|
| 2830 |
+
"gflops": 40156.457318985966,
|
| 2831 |
+
"ms": 4104.128519691438
|
| 2832 |
+
},
|
| 2833 |
+
{
|
| 2834 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2835 |
+
"method": "confidence",
|
| 2836 |
+
"metric": 0.18066434967152634,
|
| 2837 |
+
"calls": 0.42676431424766975,
|
| 2838 |
+
"gflops": 41478.6488018812,
|
| 2839 |
+
"ms": 4238.190274278827
|
| 2840 |
+
},
|
| 2841 |
+
{
|
| 2842 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2843 |
+
"method": "confidence",
|
| 2844 |
+
"metric": 0.1803913789289444,
|
| 2845 |
+
"calls": 0.4324234354194407,
|
| 2846 |
+
"gflops": 42026.874538691416,
|
| 2847 |
+
"ms": 4293.776855449209
|
| 2848 |
+
},
|
| 2849 |
+
{
|
| 2850 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2851 |
+
"method": "confidence",
|
| 2852 |
+
"metric": 0.17959468902865128,
|
| 2853 |
+
"calls": 0.44107856191744343,
|
| 2854 |
+
"gflops": 42865.33743028352,
|
| 2855 |
+
"ms": 4378.791626650968
|
| 2856 |
+
},
|
| 2857 |
+
{
|
| 2858 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2859 |
+
"method": "confidence",
|
| 2860 |
+
"metric": 0.17863383514615866,
|
| 2861 |
+
"calls": 0.45772303595206393,
|
| 2862 |
+
"gflops": 44477.766067960634,
|
| 2863 |
+
"ms": 4542.281571269737
|
| 2864 |
+
},
|
| 2865 |
+
{
|
| 2866 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2867 |
+
"method": "confidence",
|
| 2868 |
+
"metric": 0.17957308538379646,
|
| 2869 |
+
"calls": 0.4717043941411451,
|
| 2870 |
+
"gflops": 45832.20612360941,
|
| 2871 |
+
"ms": 4679.6131247495005
|
| 2872 |
+
},
|
| 2873 |
+
{
|
| 2874 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2875 |
+
"method": "confidence",
|
| 2876 |
+
"metric": 0.17910245145856155,
|
| 2877 |
+
"calls": 0.4826897470039947,
|
| 2878 |
+
"gflops": 46896.40902447631,
|
| 2879 |
+
"ms": 4787.516488197888
|
| 2880 |
+
},
|
| 2881 |
+
{
|
| 2882 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2883 |
+
"method": "confidence",
|
| 2884 |
+
"metric": 0.17871833907402393,
|
| 2885 |
+
"calls": 0.4896804260985353,
|
| 2886 |
+
"gflops": 47573.62905230069,
|
| 2887 |
+
"ms": 4856.182264937771
|
| 2888 |
+
},
|
| 2889 |
+
{
|
| 2890 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2891 |
+
"method": "confidence",
|
| 2892 |
+
"metric": 0.18015785955296285,
|
| 2893 |
+
"calls": 0.5013315579227696,
|
| 2894 |
+
"gflops": 48702.32909867467,
|
| 2895 |
+
"ms": 4970.625226170908
|
| 2896 |
+
},
|
| 2897 |
+
{
|
| 2898 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2899 |
+
"method": "confidence",
|
| 2900 |
+
"metric": 0.1810878167493375,
|
| 2901 |
+
"calls": 0.5116511318242344,
|
| 2902 |
+
"gflops": 49702.034854034486,
|
| 2903 |
+
"ms": 5071.988991834544
|
| 2904 |
+
},
|
| 2905 |
+
{
|
| 2906 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2907 |
+
"method": "confidence",
|
| 2908 |
+
"metric": 0.18177962418520882,
|
| 2909 |
+
"calls": 0.5183089214380826,
|
| 2910 |
+
"gflops": 50347.00630910533,
|
| 2911 |
+
"ms": 5137.384969682052
|
| 2912 |
+
},
|
| 2913 |
+
{
|
| 2914 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2915 |
+
"method": "confidence",
|
| 2916 |
+
"metric": 0.18246323539519374,
|
| 2917 |
+
"calls": 0.5252996005326231,
|
| 2918 |
+
"gflops": 51024.226336929714,
|
| 2919 |
+
"ms": 5206.050746421934
|
| 2920 |
+
},
|
| 2921 |
+
{
|
| 2922 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2923 |
+
"method": "confidence",
|
| 2924 |
+
"metric": 0.1839876166906015,
|
| 2925 |
+
"calls": 0.5396138482023968,
|
| 2926 |
+
"gflops": 52410.914965332035,
|
| 2927 |
+
"ms": 5346.652098794074
|
| 2928 |
+
},
|
| 2929 |
+
{
|
| 2930 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2931 |
+
"method": "confidence",
|
| 2932 |
+
"metric": 0.1846674213907964,
|
| 2933 |
+
"calls": 0.5452729693741678,
|
| 2934 |
+
"gflops": 52959.14070214226,
|
| 2935 |
+
"ms": 5402.2386799644555
|
| 2936 |
+
},
|
| 2937 |
+
{
|
| 2938 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2939 |
+
"method": "confidence",
|
| 2940 |
+
"metric": 0.18676294233791663,
|
| 2941 |
+
"calls": 0.559254327563249,
|
| 2942 |
+
"gflops": 54313.58075779103,
|
| 2943 |
+
"ms": 5539.57023344422
|
| 2944 |
+
},
|
| 2945 |
+
{
|
| 2946 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2947 |
+
"method": "confidence",
|
| 2948 |
+
"metric": 0.18938663221175195,
|
| 2949 |
+
"calls": 0.572237017310253,
|
| 2950 |
+
"gflops": 55571.275095179175,
|
| 2951 |
+
"ms": 5667.092390246859
|
| 2952 |
+
},
|
| 2953 |
+
{
|
| 2954 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2955 |
+
"method": "confidence",
|
| 2956 |
+
"metric": 0.1909571002676625,
|
| 2957 |
+
"calls": 0.5862183754993342,
|
| 2958 |
+
"gflops": 56925.71515082795,
|
| 2959 |
+
"ms": 5804.423943726624
|
| 2960 |
+
},
|
| 2961 |
+
{
|
| 2962 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2963 |
+
"method": "confidence",
|
| 2964 |
+
"metric": 0.1934300101790686,
|
| 2965 |
+
"calls": 0.5998668442077231,
|
| 2966 |
+
"gflops": 58247.90663372319,
|
| 2967 |
+
"ms": 5938.485698314014
|
| 2968 |
+
},
|
| 2969 |
+
{
|
| 2970 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2971 |
+
"method": "confidence",
|
| 2972 |
+
"metric": 0.1973701682130997,
|
| 2973 |
+
"calls": 0.6191744340878829,
|
| 2974 |
+
"gflops": 60118.32385342864,
|
| 2975 |
+
"ms": 6128.134034071785
|
| 2976 |
+
},
|
| 2977 |
+
{
|
| 2978 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2979 |
+
"method": "confidence",
|
| 2980 |
+
"metric": 0.1986727924998457,
|
| 2981 |
+
"calls": 0.6264980026631158,
|
| 2982 |
+
"gflops": 60827.792454006565,
|
| 2983 |
+
"ms": 6200.069609704043
|
| 2984 |
+
},
|
| 2985 |
+
{
|
| 2986 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2987 |
+
"method": "confidence",
|
| 2988 |
+
"metric": 0.20186289191698192,
|
| 2989 |
+
"calls": 0.6384820239680427,
|
| 2990 |
+
"gflops": 61988.74107313409,
|
| 2991 |
+
"ms": 6317.782369829556
|
| 2992 |
+
},
|
| 2993 |
+
{
|
| 2994 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 2995 |
+
"method": "confidence",
|
| 2996 |
+
"metric": 0.20246031058520178,
|
| 2997 |
+
"calls": 0.6461384820239681,
|
| 2998 |
+
"gflops": 62730.45824646557,
|
| 2999 |
+
"ms": 6392.987744354189
|
| 3000 |
+
},
|
| 3001 |
+
{
|
| 3002 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3003 |
+
"method": "confidence",
|
| 3004 |
+
"metric": 0.20349944137622245,
|
| 3005 |
+
"calls": 0.653462050599201,
|
| 3006 |
+
"gflops": 63439.92684704349,
|
| 3007 |
+
"ms": 6464.923319986447
|
| 3008 |
+
},
|
| 3009 |
+
{
|
| 3010 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3011 |
+
"method": "confidence",
|
| 3012 |
+
"metric": 0.20442990090722246,
|
| 3013 |
+
"calls": 0.6597869507323568,
|
| 3014 |
+
"gflops": 64052.64972936079,
|
| 3015 |
+
"ms": 6527.049498941578
|
| 3016 |
+
},
|
| 3017 |
+
{
|
| 3018 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3019 |
+
"method": "confidence",
|
| 3020 |
+
"metric": 0.2044993942500819,
|
| 3021 |
+
"calls": 0.6661118508655126,
|
| 3022 |
+
"gflops": 64665.372611678096,
|
| 3023 |
+
"ms": 6589.17567789671
|
| 3024 |
+
},
|
| 3025 |
+
{
|
| 3026 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3027 |
+
"method": "confidence",
|
| 3028 |
+
"metric": 0.2064379542733177,
|
| 3029 |
+
"calls": 0.6734354194407457,
|
| 3030 |
+
"gflops": 65374.84121225603,
|
| 3031 |
+
"ms": 6661.1112535289685
|
| 3032 |
+
},
|
| 3033 |
+
{
|
| 3034 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3035 |
+
"method": "confidence",
|
| 3036 |
+
"metric": 0.2065831206227621,
|
| 3037 |
+
"calls": 0.6820905459387483,
|
| 3038 |
+
"gflops": 66213.30410384812,
|
| 3039 |
+
"ms": 6746.126024730727
|
| 3040 |
+
},
|
| 3041 |
+
{
|
| 3042 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3043 |
+
"method": "confidence",
|
| 3044 |
+
"metric": 0.20802802394105854,
|
| 3045 |
+
"calls": 0.6927430093209055,
|
| 3046 |
+
"gflops": 67245.25843196148,
|
| 3047 |
+
"ms": 6850.759589286739
|
| 3048 |
+
},
|
| 3049 |
+
{
|
| 3050 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3051 |
+
"method": "confidence",
|
| 3052 |
+
"metric": 0.21169680596278434,
|
| 3053 |
+
"calls": 0.7090545938748336,
|
| 3054 |
+
"gflops": 68825.43849688505,
|
| 3055 |
+
"ms": 7010.979735013132
|
| 3056 |
+
},
|
| 3057 |
+
{
|
| 3058 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3059 |
+
"method": "confidence",
|
| 3060 |
+
"metric": 0.2139280527103987,
|
| 3061 |
+
"calls": 0.7170439414114514,
|
| 3062 |
+
"gflops": 69599.40424297006,
|
| 3063 |
+
"ms": 7089.454908430141
|
| 3064 |
+
},
|
| 3065 |
+
{
|
| 3066 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3067 |
+
"method": "confidence",
|
| 3068 |
+
"metric": 0.2146900231025185,
|
| 3069 |
+
"calls": 0.7243675099866844,
|
| 3070 |
+
"gflops": 70308.87284354799,
|
| 3071 |
+
"ms": 7161.390484062398
|
| 3072 |
+
},
|
| 3073 |
+
{
|
| 3074 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3075 |
+
"method": "confidence",
|
| 3076 |
+
"metric": 0.21653561577692484,
|
| 3077 |
+
"calls": 0.7330226364846871,
|
| 3078 |
+
"gflops": 71147.33573514009,
|
| 3079 |
+
"ms": 7246.405255264158
|
| 3080 |
+
},
|
| 3081 |
+
{
|
| 3082 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3083 |
+
"method": "confidence",
|
| 3084 |
+
"metric": 0.21721689648133474,
|
| 3085 |
+
"calls": 0.7363515312916112,
|
| 3086 |
+
"gflops": 71469.82146267552,
|
| 3087 |
+
"ms": 7279.1032441879115
|
| 3088 |
+
},
|
| 3089 |
+
{
|
| 3090 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3091 |
+
"method": "confidence",
|
| 3092 |
+
"metric": 0.22103445955201576,
|
| 3093 |
+
"calls": 0.7486684420772304,
|
| 3094 |
+
"gflops": 72663.01865455658,
|
| 3095 |
+
"ms": 7400.0858032057995
|
| 3096 |
+
},
|
| 3097 |
+
{
|
| 3098 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3099 |
+
"method": "confidence",
|
| 3100 |
+
"metric": 0.22340134248367294,
|
| 3101 |
+
"calls": 0.757323568575233,
|
| 3102 |
+
"gflops": 73501.48154614867,
|
| 3103 |
+
"ms": 7485.100574407558
|
| 3104 |
+
},
|
| 3105 |
+
{
|
| 3106 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3107 |
+
"method": "confidence",
|
| 3108 |
+
"metric": 0.2269322568854726,
|
| 3109 |
+
"calls": 0.7639813581890812,
|
| 3110 |
+
"gflops": 74146.45300121952,
|
| 3111 |
+
"ms": 7550.496552255066
|
| 3112 |
+
},
|
| 3113 |
+
{
|
| 3114 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3115 |
+
"method": "confidence",
|
| 3116 |
+
"metric": 0.22757452798038888,
|
| 3117 |
+
"calls": 0.7733022636484687,
|
| 3118 |
+
"gflops": 75049.4130383187,
|
| 3119 |
+
"ms": 7642.050921241575
|
| 3120 |
+
},
|
| 3121 |
+
{
|
| 3122 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3123 |
+
"method": "confidence",
|
| 3124 |
+
"metric": 0.227821050912831,
|
| 3125 |
+
"calls": 0.784953395472703,
|
| 3126 |
+
"gflops": 76178.11308469268,
|
| 3127 |
+
"ms": 7756.493882474713
|
| 3128 |
+
},
|
| 3129 |
+
{
|
| 3130 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3131 |
+
"method": "confidence",
|
| 3132 |
+
"metric": 0.22991865671924006,
|
| 3133 |
+
"calls": 0.7939414114513982,
|
| 3134 |
+
"gflops": 77048.82454903833,
|
| 3135 |
+
"ms": 7844.7784525688485
|
| 3136 |
+
},
|
| 3137 |
+
{
|
| 3138 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3139 |
+
"method": "confidence",
|
| 3140 |
+
"metric": 0.23486191981046528,
|
| 3141 |
+
"calls": 0.8039280958721704,
|
| 3142 |
+
"gflops": 78016.28173164459,
|
| 3143 |
+
"ms": 7942.8724193401085
|
| 3144 |
+
},
|
| 3145 |
+
{
|
| 3146 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3147 |
+
"method": "confidence",
|
| 3148 |
+
"metric": 0.23803916047544685,
|
| 3149 |
+
"calls": 0.8122503328894807,
|
| 3150 |
+
"gflops": 78822.49605048315,
|
| 3151 |
+
"ms": 8024.617391649493
|
| 3152 |
+
},
|
| 3153 |
+
{
|
| 3154 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3155 |
+
"method": "confidence",
|
| 3156 |
+
"metric": 0.24022947987632814,
|
| 3157 |
+
"calls": 0.8215712383488681,
|
| 3158 |
+
"gflops": 79725.45608758234,
|
| 3159 |
+
"ms": 8116.171760636003
|
| 3160 |
+
},
|
| 3161 |
+
{
|
| 3162 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3163 |
+
"method": "confidence",
|
| 3164 |
+
"metric": 0.24528627405740036,
|
| 3165 |
+
"calls": 0.8388814913448736,
|
| 3166 |
+
"gflops": 81402.38187076653,
|
| 3167 |
+
"ms": 8286.201303039523
|
| 3168 |
+
},
|
| 3169 |
+
{
|
| 3170 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3171 |
+
"method": "confidence",
|
| 3172 |
+
"metric": 0.249537136326177,
|
| 3173 |
+
"calls": 0.8535286284953395,
|
| 3174 |
+
"gflops": 82821.3190719224,
|
| 3175 |
+
"ms": 8430.072454304036
|
| 3176 |
+
},
|
| 3177 |
+
{
|
| 3178 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3179 |
+
"method": "confidence",
|
| 3180 |
+
"metric": 0.25211074301800224,
|
| 3181 |
+
"calls": 0.8638482023968043,
|
| 3182 |
+
"gflops": 83821.0248272822,
|
| 3183 |
+
"ms": 8531.436219967674
|
| 3184 |
+
},
|
| 3185 |
+
{
|
| 3186 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3187 |
+
"method": "confidence",
|
| 3188 |
+
"metric": 0.2540042317315216,
|
| 3189 |
+
"calls": 0.8745006657789614,
|
| 3190 |
+
"gflops": 84852.97915539556,
|
| 3191 |
+
"ms": 8636.069784523685
|
| 3192 |
+
},
|
| 3193 |
+
{
|
| 3194 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3195 |
+
"method": "confidence",
|
| 3196 |
+
"metric": 0.2540996517958595,
|
| 3197 |
+
"calls": 0.8764980026631158,
|
| 3198 |
+
"gflops": 85046.4705919168,
|
| 3199 |
+
"ms": 8655.688577877936
|
| 3200 |
+
},
|
| 3201 |
+
{
|
| 3202 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3203 |
+
"method": "confidence",
|
| 3204 |
+
"metric": 0.25436194678023316,
|
| 3205 |
+
"calls": 0.8841544607190412,
|
| 3206 |
+
"gflops": 85788.18776524828,
|
| 3207 |
+
"ms": 8730.89395240257
|
| 3208 |
+
},
|
| 3209 |
+
{
|
| 3210 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3211 |
+
"method": "confidence",
|
| 3212 |
+
"metric": 0.2534339447803836,
|
| 3213 |
+
"calls": 0.8891478029294274,
|
| 3214 |
+
"gflops": 86271.91635655142,
|
| 3215 |
+
"ms": 8779.9409357882
|
| 3216 |
+
},
|
| 3217 |
+
{
|
| 3218 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3219 |
+
"method": "confidence",
|
| 3220 |
+
"metric": 0.2563240880787248,
|
| 3221 |
+
"calls": 0.8981358189081226,
|
| 3222 |
+
"gflops": 87142.62782089706,
|
| 3223 |
+
"ms": 8868.225505882336
|
| 3224 |
+
},
|
| 3225 |
+
{
|
| 3226 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3227 |
+
"method": "confidence",
|
| 3228 |
+
"metric": 0.25551845731687656,
|
| 3229 |
+
"calls": 0.9054593874833555,
|
| 3230 |
+
"gflops": 87852.09642147498,
|
| 3231 |
+
"ms": 8940.161081514594
|
| 3232 |
+
},
|
| 3233 |
+
{
|
| 3234 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3235 |
+
"method": "confidence",
|
| 3236 |
+
"metric": 0.2597490004762105,
|
| 3237 |
+
"calls": 0.9221038615179761,
|
| 3238 |
+
"gflops": 89464.5250591521,
|
| 3239 |
+
"ms": 9103.651026133362
|
| 3240 |
+
},
|
| 3241 |
+
{
|
| 3242 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3243 |
+
"method": "confidence",
|
| 3244 |
+
"metric": 0.26306323674307786,
|
| 3245 |
+
"calls": 0.9254327563249002,
|
| 3246 |
+
"gflops": 89787.01078668753,
|
| 3247 |
+
"ms": 9136.349015057116
|
| 3248 |
+
},
|
| 3249 |
+
{
|
| 3250 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3251 |
+
"method": "confidence",
|
| 3252 |
+
"metric": 0.26218185491449153,
|
| 3253 |
+
"calls": 0.9364181091877497,
|
| 3254 |
+
"gflops": 90851.21368755442,
|
| 3255 |
+
"ms": 9244.252378505502
|
| 3256 |
+
},
|
| 3257 |
+
{
|
| 3258 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3259 |
+
"method": "confidence",
|
| 3260 |
+
"metric": 0.26357052137555514,
|
| 3261 |
+
"calls": 0.953728362183755,
|
| 3262 |
+
"gflops": 92528.13947073862,
|
| 3263 |
+
"ms": 9414.281920909021
|
| 3264 |
+
},
|
| 3265 |
+
{
|
| 3266 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3267 |
+
"method": "confidence",
|
| 3268 |
+
"metric": 0.26400290307863034,
|
| 3269 |
+
"calls": 0.96271637816245,
|
| 3270 |
+
"gflops": 93398.85093508427,
|
| 3271 |
+
"ms": 9502.566491003154
|
| 3272 |
+
},
|
| 3273 |
+
{
|
| 3274 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3275 |
+
"method": "confidence",
|
| 3276 |
+
"metric": 0.2647475724296899,
|
| 3277 |
+
"calls": 0.9700399467376831,
|
| 3278 |
+
"gflops": 94108.3195356622,
|
| 3279 |
+
"ms": 9574.502066635414
|
| 3280 |
+
},
|
| 3281 |
+
{
|
| 3282 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3283 |
+
"method": "confidence",
|
| 3284 |
+
"metric": 0.2649306125590882,
|
| 3285 |
+
"calls": 0.987017310252996,
|
| 3286 |
+
"gflops": 95752.99674609285,
|
| 3287 |
+
"ms": 9741.261810146556
|
| 3288 |
+
},
|
| 3289 |
+
{
|
| 3290 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 3291 |
+
"method": "confidence",
|
| 3292 |
+
"metric": 0.27001009067886245,
|
| 3293 |
+
"calls": 1.0,
|
| 3294 |
+
"gflops": 97010.691083481,
|
| 3295 |
+
"ms": 9868.783966949195
|
| 3296 |
+
}
|
| 3297 |
+
],
|
| 3298 |
+
"protocol": "Five-seed mean call rates times existing measured mean component costs; not integrated latency measurements."
|
| 3299 |
+
}
|
releases/20260913_lr/paper_data/earlier_results.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
releases/20260913_lr/paper_data/fig2_audit/remote_counts.json
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"expert": "yolo26x",
|
| 4 |
+
"endpoint": "MiniCPM",
|
| 5 |
+
"images": 3004,
|
| 6 |
+
"counts": {
|
| 7 |
+
"both_correct": 8176,
|
| 8 |
+
"negative": 11580,
|
| 9 |
+
"both_correct_negative": 7940,
|
| 10 |
+
"rescuable": 1198,
|
| 11 |
+
"harmful": 2109,
|
| 12 |
+
"positive": 436,
|
| 13 |
+
"both_wrong": 533,
|
| 14 |
+
"both_correct_positive": 236,
|
| 15 |
+
"null_predictions": 112,
|
| 16 |
+
"metric_vs_state_difference": 103,
|
| 17 |
+
"difference_null": 103
|
| 18 |
+
}
|
| 19 |
+
},
|
| 20 |
+
{
|
| 21 |
+
"expert": "yolo26x",
|
| 22 |
+
"endpoint": "Qwen3.8",
|
| 23 |
+
"images": 3004,
|
| 24 |
+
"counts": {
|
| 25 |
+
"both_correct": 9508,
|
| 26 |
+
"negative": 11580,
|
| 27 |
+
"both_correct_negative": 9198,
|
| 28 |
+
"rescuable": 1282,
|
| 29 |
+
"positive": 436,
|
| 30 |
+
"both_correct_positive": 310,
|
| 31 |
+
"both_wrong": 449,
|
| 32 |
+
"harmful": 777,
|
| 33 |
+
"null_predictions": 36,
|
| 34 |
+
"metric_vs_state_difference": 34,
|
| 35 |
+
"difference_null": 34
|
| 36 |
+
}
|
| 37 |
+
},
|
| 38 |
+
{
|
| 39 |
+
"expert": "rtdetr_x_fp32",
|
| 40 |
+
"endpoint": "MiniCPM",
|
| 41 |
+
"images": 3004,
|
| 42 |
+
"counts": {
|
| 43 |
+
"rescuable": 3314,
|
| 44 |
+
"negative": 11580,
|
| 45 |
+
"both_correct": 6060,
|
| 46 |
+
"both_correct_negative": 5846,
|
| 47 |
+
"harmful": 1945,
|
| 48 |
+
"positive": 436,
|
| 49 |
+
"both_wrong": 697,
|
| 50 |
+
"both_correct_positive": 214,
|
| 51 |
+
"null_predictions": 112,
|
| 52 |
+
"metric_vs_state_difference": 103,
|
| 53 |
+
"difference_null": 103
|
| 54 |
+
}
|
| 55 |
+
},
|
| 56 |
+
{
|
| 57 |
+
"expert": "rtdetr_x_fp32",
|
| 58 |
+
"endpoint": "Qwen3.8",
|
| 59 |
+
"images": 3004,
|
| 60 |
+
"counts": {
|
| 61 |
+
"rescuable": 3461,
|
| 62 |
+
"negative": 11580,
|
| 63 |
+
"both_correct": 7329,
|
| 64 |
+
"both_correct_negative": 7036,
|
| 65 |
+
"positive": 436,
|
| 66 |
+
"both_correct_positive": 293,
|
| 67 |
+
"both_wrong": 550,
|
| 68 |
+
"harmful": 676,
|
| 69 |
+
"null_predictions": 36,
|
| 70 |
+
"metric_vs_state_difference": 34,
|
| 71 |
+
"difference_null": 34
|
| 72 |
+
}
|
| 73 |
+
}
|
| 74 |
+
]
|
releases/20260913_lr/paper_data/fig2_audit/remote_counts.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pathlib import Path
|
| 2 |
+
import json
|
| 3 |
+
from collections import Counter
|
| 4 |
+
|
| 5 |
+
base=Path('/mnt/HithinkOmniSSD/user_workspace/caisihang/research/ICASSP_ModelCollaboration/outputs/router_full_20260910/data')
|
| 6 |
+
out=[]
|
| 7 |
+
for expert in ['yolo26x','rtdetr_x_fp32']:
|
| 8 |
+
rows=[json.loads(s) for s in (base/expert/'test/records.jsonl').read_text().splitlines()]
|
| 9 |
+
for endpoint in ['MiniCPM','Qwen3.8']:
|
| 10 |
+
c=Counter()
|
| 11 |
+
for r in rows:
|
| 12 |
+
for event,s in r['small_label']['events'].items():
|
| 13 |
+
l=r['large_labels'][endpoint]['events'][event]
|
| 14 |
+
assert s['target']==l['target']
|
| 15 |
+
state={(True,True):'both_correct',(False,True):'rescuable',(True,False):'harmful',(False,False):'both_wrong'}[(bool(s['correct']),bool(l['correct']))]
|
| 16 |
+
c[state]+=1
|
| 17 |
+
c['positive' if s['target'] else 'negative']+=1
|
| 18 |
+
if state=='both_correct':c['both_correct_positive' if s['target'] else 'both_correct_negative']+=1
|
| 19 |
+
if l['prediction'] is None:c['null_predictions']+=1
|
| 20 |
+
score_correct=(l['prediction'] is True)==bool(l['target'])
|
| 21 |
+
if score_correct!=bool(l['correct']):
|
| 22 |
+
c['metric_vs_state_difference']+=1
|
| 23 |
+
c['difference_null' if l['prediction'] is None else 'difference_other']+=1
|
| 24 |
+
out.append(dict(expert=expert,endpoint=endpoint,images=len(rows),counts=dict(c)))
|
| 25 |
+
print(json.dumps(out,indent=2))
|
releases/20260913_lr/paper_data/five_seed_counts.json
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"better_metric": 11,
|
| 3 |
+
"fewer_calls": 9,
|
| 4 |
+
"both": 8,
|
| 5 |
+
"above_endpoints": 10,
|
| 6 |
+
"better_ranked_area": 16
|
| 7 |
+
}
|
releases/20260913_lr/paper_data/full_budget_results.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
releases/20260913_lr/paper_data/historical_results_v44.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
releases/20260913_lr/paper_data/joint_MiniCPM_comparison_20260910.json
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"expert": "cub",
|
| 4 |
+
"endpoint": "MiniCPM",
|
| 5 |
+
"single_task_test_fixed25": 0.7119433897134967,
|
| 6 |
+
"multi_task_test_fixed25": 0.7079737659647911,
|
| 7 |
+
"delta_multi_minus_single": -0.003969623748705575
|
| 8 |
+
},
|
| 9 |
+
{
|
| 10 |
+
"expert": "glsim",
|
| 11 |
+
"endpoint": "MiniCPM",
|
| 12 |
+
"single_task_test_fixed25": 0.759233690024163,
|
| 13 |
+
"multi_task_test_fixed25": 0.7745944080082844,
|
| 14 |
+
"delta_multi_minus_single": 0.01536071798412142
|
| 15 |
+
},
|
| 16 |
+
{
|
| 17 |
+
"expert": "nlvr2",
|
| 18 |
+
"endpoint": "MiniCPM",
|
| 19 |
+
"single_task_test_fixed25": 0.8346846523642104,
|
| 20 |
+
"multi_task_test_fixed25": 0.8309535768099304,
|
| 21 |
+
"delta_multi_minus_single": -0.0037310755542799834
|
| 22 |
+
},
|
| 23 |
+
{
|
| 24 |
+
"expert": "vilt",
|
| 25 |
+
"endpoint": "MiniCPM",
|
| 26 |
+
"single_task_test_fixed25": 0.8046925450240368,
|
| 27 |
+
"multi_task_test_fixed25": 0.802611752887996,
|
| 28 |
+
"delta_multi_minus_single": -0.002080792136040732
|
| 29 |
+
},
|
| 30 |
+
{
|
| 31 |
+
"expert": "groundingdino",
|
| 32 |
+
"endpoint": "MiniCPM",
|
| 33 |
+
"single_task_test_fixed25": 0.3598105663159686,
|
| 34 |
+
"multi_task_test_fixed25": 0.37004792558772653,
|
| 35 |
+
"delta_multi_minus_single": 0.010237359271757951
|
| 36 |
+
},
|
| 37 |
+
{
|
| 38 |
+
"expert": "instancevg",
|
| 39 |
+
"endpoint": "MiniCPM",
|
| 40 |
+
"single_task_test_fixed25": 0.6394237586138446,
|
| 41 |
+
"multi_task_test_fixed25": 0.6243654822335025,
|
| 42 |
+
"delta_multi_minus_single": -0.015058276380342073
|
| 43 |
+
},
|
| 44 |
+
{
|
| 45 |
+
"expert": "yolo26x",
|
| 46 |
+
"endpoint": "MiniCPM",
|
| 47 |
+
"single_task_test_fixed25": 0.18032734231886507,
|
| 48 |
+
"multi_task_test_fixed25": 0.1836164473038217,
|
| 49 |
+
"delta_multi_minus_single": 0.003289104984956631
|
| 50 |
+
},
|
| 51 |
+
{
|
| 52 |
+
"expert": "rtdetr_x_fp32",
|
| 53 |
+
"endpoint": "MiniCPM",
|
| 54 |
+
"single_task_test_fixed25": 0.10367040308754236,
|
| 55 |
+
"multi_task_test_fixed25": 0.11260360812088349,
|
| 56 |
+
"delta_multi_minus_single": 0.008933205033341135
|
| 57 |
+
}
|
| 58 |
+
]
|
releases/20260913_lr/paper_data/joint_Qwen3.8_comparison_20260910.json
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"expert": "cub",
|
| 4 |
+
"endpoint": "Qwen3.8",
|
| 5 |
+
"single_task_test_fixed25": 0.7619951674145667,
|
| 6 |
+
"multi_task_test_fixed25": 0.7782188470831896,
|
| 7 |
+
"delta_multi_minus_single": 0.016223679668622815
|
| 8 |
+
},
|
| 9 |
+
{
|
| 10 |
+
"expert": "glsim",
|
| 11 |
+
"endpoint": "Qwen3.8",
|
| 12 |
+
"single_task_test_fixed25": 0.8301691404901622,
|
| 13 |
+
"multi_task_test_fixed25": 0.8448394891266828,
|
| 14 |
+
"delta_multi_minus_single": 0.014670348636520614
|
| 15 |
+
},
|
| 16 |
+
{
|
| 17 |
+
"expert": "nlvr2",
|
| 18 |
+
"endpoint": "Qwen3.8",
|
| 19 |
+
"single_task_test_fixed25": 0.8372677046710196,
|
| 20 |
+
"multi_task_test_fixed25": 0.8337518834756403,
|
| 21 |
+
"delta_multi_minus_single": -0.0035158211953792984
|
| 22 |
+
},
|
| 23 |
+
{
|
| 24 |
+
"expert": "vilt",
|
| 25 |
+
"endpoint": "Qwen3.8",
|
| 26 |
+
"single_task_test_fixed25": 0.8045490421181029,
|
| 27 |
+
"multi_task_test_fixed25": 0.8067733371600775,
|
| 28 |
+
"delta_multi_minus_single": 0.0022242950419746332
|
| 29 |
+
},
|
| 30 |
+
{
|
| 31 |
+
"expert": "groundingdino",
|
| 32 |
+
"endpoint": "Qwen3.8",
|
| 33 |
+
"single_task_test_fixed25": 0.4223974137197629,
|
| 34 |
+
"multi_task_test_fixed25": 0.45991549215892014,
|
| 35 |
+
"delta_multi_minus_single": 0.03751807843915722
|
| 36 |
+
},
|
| 37 |
+
{
|
| 38 |
+
"expert": "instancevg",
|
| 39 |
+
"endpoint": "Qwen3.8",
|
| 40 |
+
"single_task_test_fixed25": 0.7224853245611548,
|
| 41 |
+
"multi_task_test_fixed25": 0.7053001729858492,
|
| 42 |
+
"delta_multi_minus_single": -0.017185151575305535
|
| 43 |
+
},
|
| 44 |
+
{
|
| 45 |
+
"expert": "yolo26x",
|
| 46 |
+
"endpoint": "Qwen3.8",
|
| 47 |
+
"single_task_test_fixed25": 0.19009376433100783,
|
| 48 |
+
"multi_task_test_fixed25": 0.19603467084956394,
|
| 49 |
+
"delta_multi_minus_single": 0.0059409065185561105
|
| 50 |
+
},
|
| 51 |
+
{
|
| 52 |
+
"expert": "rtdetr_x_fp32",
|
| 53 |
+
"endpoint": "Qwen3.8",
|
| 54 |
+
"single_task_test_fixed25": 0.10483176081963008,
|
| 55 |
+
"multi_task_test_fixed25": 0.12038495915476813,
|
| 56 |
+
"delta_multi_minus_single": 0.015553198335138047
|
| 57 |
+
}
|
| 58 |
+
]
|
releases/20260913_lr/paper_data/layout_verification.json
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"old_distribution": {
|
| 3 |
+
"width_pt": 523.1995239257812,
|
| 4 |
+
"height_pt": 309.7997741699219,
|
| 5 |
+
"height_at_7_16_in": 4.239618504261674
|
| 6 |
+
},
|
| 7 |
+
"old_budget": {
|
| 8 |
+
"width_pt": 526.7994995117188,
|
| 9 |
+
"height_pt": 237.7997589111328,
|
| 10 |
+
"height_at_7_16_in": 3.2320575007794505
|
| 11 |
+
},
|
| 12 |
+
"combined": {
|
| 13 |
+
"width_pt": 515.52001953125,
|
| 14 |
+
"height_pt": 324.0,
|
| 15 |
+
"height_at_7_16_in": 4.499999829510743
|
| 16 |
+
},
|
| 17 |
+
"reduction_percent": 39.77255134624947,
|
| 18 |
+
"old_pages": 16,
|
| 19 |
+
"new_pages": 14,
|
| 20 |
+
"new_figure_and_section_pages": [
|
| 21 |
+
6,
|
| 22 |
+
7
|
| 23 |
+
],
|
| 24 |
+
"changed_sections": [
|
| 25 |
+
"sections\\appendix.tex",
|
| 26 |
+
"sections\\experiments.tex"
|
| 27 |
+
],
|
| 28 |
+
"figure1_identical": true
|
| 29 |
+
}
|
releases/20260913_lr/paper_data/lr_control_curves.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
releases/20260913_lr/paper_data/lr_control_summary.json
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"4S_difference": {
|
| 3 |
+
"cub": {
|
| 4 |
+
"full": 71.30477624717177,
|
| 5 |
+
"middle_high": 66.65309176574908
|
| 6 |
+
},
|
| 7 |
+
"grefcoco": {
|
| 8 |
+
"full": 60.10334108982569,
|
| 9 |
+
"middle_high": 62.55288392181684
|
| 10 |
+
},
|
| 11 |
+
"nlvr2": {
|
| 12 |
+
"full": 86.42688661827373,
|
| 13 |
+
"middle_high": 87.82059502279398
|
| 14 |
+
},
|
| 15 |
+
"construction": {
|
| 16 |
+
"full": 21.68664457361428,
|
| 17 |
+
"middle_high": 23.37179473046792
|
| 18 |
+
},
|
| 19 |
+
"overall": {
|
| 20 |
+
"mean": 59.880412132221366,
|
| 21 |
+
"std": 0.06230299379475004
|
| 22 |
+
}
|
| 23 |
+
},
|
| 24 |
+
"4S_RH": {
|
| 25 |
+
"cub": {
|
| 26 |
+
"full": 67.80147679642886,
|
| 27 |
+
"middle_high": 62.716073314651304
|
| 28 |
+
},
|
| 29 |
+
"grefcoco": {
|
| 30 |
+
"full": 59.83696029450736,
|
| 31 |
+
"middle_high": 62.2898964392558
|
| 32 |
+
},
|
| 33 |
+
"nlvr2": {
|
| 34 |
+
"full": 87.36051668387572,
|
| 35 |
+
"middle_high": 89.08451490248436
|
| 36 |
+
},
|
| 37 |
+
"construction": {
|
| 38 |
+
"full": 21.11319259968111,
|
| 39 |
+
"middle_high": 22.83140793334926
|
| 40 |
+
},
|
| 41 |
+
"overall": {
|
| 42 |
+
"mean": 59.028036593623256,
|
| 43 |
+
"std": 0.09215446832595518
|
| 44 |
+
}
|
| 45 |
+
},
|
| 46 |
+
"4S_error_ratio": {
|
| 47 |
+
"cub": {
|
| 48 |
+
"full": 67.63433680158339,
|
| 49 |
+
"middle_high": 62.495205558823486
|
| 50 |
+
},
|
| 51 |
+
"grefcoco": {
|
| 52 |
+
"full": 59.9030469001769,
|
| 53 |
+
"middle_high": 62.255919404292584
|
| 54 |
+
},
|
| 55 |
+
"nlvr2": {
|
| 56 |
+
"full": 87.36099315635701,
|
| 57 |
+
"middle_high": 89.08882695682252
|
| 58 |
+
},
|
| 59 |
+
"construction": {
|
| 60 |
+
"full": 21.29756392944788,
|
| 61 |
+
"middle_high": 22.972511633206114
|
| 62 |
+
},
|
| 63 |
+
"overall": {
|
| 64 |
+
"mean": 59.048985196891294,
|
| 65 |
+
"std": 0.10043056570886838
|
| 66 |
+
}
|
| 67 |
+
},
|
| 68 |
+
"4S_normalized": {
|
| 69 |
+
"cub": {
|
| 70 |
+
"full": 71.3715854777698,
|
| 71 |
+
"middle_high": 66.74755638479657
|
| 72 |
+
},
|
| 73 |
+
"grefcoco": {
|
| 74 |
+
"full": 60.65894058616111,
|
| 75 |
+
"middle_high": 63.22949855130163
|
| 76 |
+
},
|
| 77 |
+
"nlvr2": {
|
| 78 |
+
"full": 87.36085314912711,
|
| 79 |
+
"middle_high": 89.0886402849806
|
| 80 |
+
},
|
| 81 |
+
"construction": {
|
| 82 |
+
"full": 21.625047131135076,
|
| 83 |
+
"middle_high": 23.29240515579527
|
| 84 |
+
},
|
| 85 |
+
"overall": {
|
| 86 |
+
"mean": 60.25410658604827,
|
| 87 |
+
"std": 0.055784402767809056
|
| 88 |
+
}
|
| 89 |
+
},
|
| 90 |
+
"three": {
|
| 91 |
+
"cub": {
|
| 92 |
+
"full": 71.27395612317933,
|
| 93 |
+
"middle_high": 66.60007988650366
|
| 94 |
+
},
|
| 95 |
+
"grefcoco": {
|
| 96 |
+
"full": 60.12755657790998,
|
| 97 |
+
"middle_high": 62.59349605468297
|
| 98 |
+
},
|
| 99 |
+
"nlvr2": {
|
| 100 |
+
"full": 86.340413568538,
|
| 101 |
+
"middle_high": 87.71263160064547
|
| 102 |
+
},
|
| 103 |
+
"construction": {
|
| 104 |
+
"full": 21.76578337259674,
|
| 105 |
+
"middle_high": 23.45434250634451
|
| 106 |
+
},
|
| 107 |
+
"overall": {
|
| 108 |
+
"mean": 59.876927410556014,
|
| 109 |
+
"std": 0.06459211304335916
|
| 110 |
+
}
|
| 111 |
+
},
|
| 112 |
+
"gain": {
|
| 113 |
+
"cub": {
|
| 114 |
+
"full": 71.01409198260043,
|
| 115 |
+
"middle_high": 66.28415678198927
|
| 116 |
+
},
|
| 117 |
+
"grefcoco": {
|
| 118 |
+
"full": 59.96696456319969,
|
| 119 |
+
"middle_high": 62.41541160228904
|
| 120 |
+
},
|
| 121 |
+
"nlvr2": {
|
| 122 |
+
"full": 86.29461067562258,
|
| 123 |
+
"middle_high": 87.67252770031322
|
| 124 |
+
},
|
| 125 |
+
"construction": {
|
| 126 |
+
"full": 21.60783215955454,
|
| 127 |
+
"middle_high": 23.264626260993804
|
| 128 |
+
},
|
| 129 |
+
"overall": {
|
| 130 |
+
"mean": 59.72087484524432,
|
| 131 |
+
"std": 0.039237511453832113
|
| 132 |
+
}
|
| 133 |
+
},
|
| 134 |
+
"posthoc": {
|
| 135 |
+
"cub": {
|
| 136 |
+
"full": 65.85980220810953,
|
| 137 |
+
"middle_high": 60.37970343902742
|
| 138 |
+
},
|
| 139 |
+
"grefcoco": {
|
| 140 |
+
"full": 59.367755773800184,
|
| 141 |
+
"middle_high": 61.58463842152389
|
| 142 |
+
},
|
| 143 |
+
"nlvr2": {
|
| 144 |
+
"full": 87.32516457516526,
|
| 145 |
+
"middle_high": 89.03494498253306
|
| 146 |
+
},
|
| 147 |
+
"construction": {
|
| 148 |
+
"full": 21.23728513819176,
|
| 149 |
+
"middle_high": 22.948980499229855
|
| 150 |
+
},
|
| 151 |
+
"overall": {
|
| 152 |
+
"mean": 58.44750192381669,
|
| 153 |
+
"std": 0.0661360515853849
|
| 154 |
+
}
|
| 155 |
+
}
|
| 156 |
+
}
|
releases/20260913_lr/paper_data/lr_main_summary.json
ADDED
|
@@ -0,0 +1,210 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"key": [
|
| 4 |
+
"cub",
|
| 5 |
+
"cub",
|
| 6 |
+
"MiniCPM"
|
| 7 |
+
],
|
| 8 |
+
"confidence_score": 81.429064549534,
|
| 9 |
+
"confidence_calls": 0.0,
|
| 10 |
+
"lr_score": 81.42906454953402,
|
| 11 |
+
"lr_calls": 0.5937176389368313,
|
| 12 |
+
"rescued": 4.6,
|
| 13 |
+
"harmed": 4.6
|
| 14 |
+
},
|
| 15 |
+
{
|
| 16 |
+
"key": [
|
| 17 |
+
"cub",
|
| 18 |
+
"cub",
|
| 19 |
+
"Qwen3.8"
|
| 20 |
+
],
|
| 21 |
+
"confidence_score": 81.429064549534,
|
| 22 |
+
"confidence_calls": 0.0,
|
| 23 |
+
"lr_score": 81.42906454953402,
|
| 24 |
+
"lr_calls": 0.0,
|
| 25 |
+
"rescued": 0.0,
|
| 26 |
+
"harmed": 0.0
|
| 27 |
+
},
|
| 28 |
+
{
|
| 29 |
+
"key": [
|
| 30 |
+
"cub",
|
| 31 |
+
"glsim",
|
| 32 |
+
"MiniCPM"
|
| 33 |
+
],
|
| 34 |
+
"confidence_score": 90.16223679668623,
|
| 35 |
+
"confidence_calls": 0.0,
|
| 36 |
+
"lr_score": 90.13117017604418,
|
| 37 |
+
"lr_calls": 0.8491542975491888,
|
| 38 |
+
"rescued": 1.0,
|
| 39 |
+
"harmed": 2.8
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"key": [
|
| 43 |
+
"cub",
|
| 44 |
+
"glsim",
|
| 45 |
+
"Qwen3.8"
|
| 46 |
+
],
|
| 47 |
+
"confidence_score": 90.16223679668623,
|
| 48 |
+
"confidence_calls": 0.0,
|
| 49 |
+
"lr_score": 90.16223679668623,
|
| 50 |
+
"lr_calls": 0.0,
|
| 51 |
+
"rescued": 0.0,
|
| 52 |
+
"harmed": 0.0
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
"key": [
|
| 56 |
+
"grefcoco",
|
| 57 |
+
"groundingdino",
|
| 58 |
+
"MiniCPM"
|
| 59 |
+
],
|
| 60 |
+
"confidence_score": 50.00708958398321,
|
| 61 |
+
"confidence_calls": 100.0,
|
| 62 |
+
"lr_score": 50.22828460425942,
|
| 63 |
+
"lr_calls": 58.18676800045374,
|
| 64 |
+
"rescued": 9475.4,
|
| 65 |
+
"harmed": 1174.4
|
| 66 |
+
},
|
| 67 |
+
{
|
| 68 |
+
"key": [
|
| 69 |
+
"grefcoco",
|
| 70 |
+
"groundingdino",
|
| 71 |
+
"Qwen3.8"
|
| 72 |
+
],
|
| 73 |
+
"confidence_score": 74.47182599325072,
|
| 74 |
+
"confidence_calls": 100.0,
|
| 75 |
+
"lr_score": 75.12179905283158,
|
| 76 |
+
"lr_calls": 88.46666477611093,
|
| 77 |
+
"rescued": 18720.8,
|
| 78 |
+
"harmed": 1641.6
|
| 79 |
+
},
|
| 80 |
+
{
|
| 81 |
+
"key": [
|
| 82 |
+
"grefcoco",
|
| 83 |
+
"instancevg",
|
| 84 |
+
"MiniCPM"
|
| 85 |
+
],
|
| 86 |
+
"confidence_score": 61.877889005473165,
|
| 87 |
+
"confidence_calls": 52.09426310864078,
|
| 88 |
+
"lr_score": 67.91367722542041,
|
| 89 |
+
"lr_calls": 19.038085245157816,
|
| 90 |
+
"rescued": 727.6,
|
| 91 |
+
"harmed": 388.2
|
| 92 |
+
},
|
| 93 |
+
{
|
| 94 |
+
"key": [
|
| 95 |
+
"grefcoco",
|
| 96 |
+
"instancevg",
|
| 97 |
+
"Qwen3.8"
|
| 98 |
+
],
|
| 99 |
+
"confidence_score": 72.43286163967898,
|
| 100 |
+
"confidence_calls": 68.38612710206165,
|
| 101 |
+
"lr_score": 76.24365482233502,
|
| 102 |
+
"lr_calls": 62.00153135014037,
|
| 103 |
+
"rescued": 4753.6,
|
| 104 |
+
"harmed": 1476.8
|
| 105 |
+
},
|
| 106 |
+
{
|
| 107 |
+
"key": [
|
| 108 |
+
"nlvr2",
|
| 109 |
+
"nlvr2",
|
| 110 |
+
"MiniCPM"
|
| 111 |
+
],
|
| 112 |
+
"confidence_score": 90.77276314845376,
|
| 113 |
+
"confidence_calls": 85.80038745784601,
|
| 114 |
+
"lr_score": 90.80002870058121,
|
| 115 |
+
"lr_calls": 91.10425486116092,
|
| 116 |
+
"rescued": 2189.4,
|
| 117 |
+
"harmed": 684.6
|
| 118 |
+
},
|
| 119 |
+
{
|
| 120 |
+
"key": [
|
| 121 |
+
"nlvr2",
|
| 122 |
+
"nlvr2",
|
| 123 |
+
"Qwen3.8"
|
| 124 |
+
],
|
| 125 |
+
"confidence_score": 90.24180239649853,
|
| 126 |
+
"confidence_calls": 85.80038745784601,
|
| 127 |
+
"lr_score": 90.60055966133314,
|
| 128 |
+
"lr_calls": 84.34096290449882,
|
| 129 |
+
"rescued": 2163.4,
|
| 130 |
+
"harmed": 686.4
|
| 131 |
+
},
|
| 132 |
+
{
|
| 133 |
+
"key": [
|
| 134 |
+
"nlvr2",
|
| 135 |
+
"vilt",
|
| 136 |
+
"MiniCPM"
|
| 137 |
+
],
|
| 138 |
+
"confidence_score": 90.64361053311329,
|
| 139 |
+
"confidence_calls": 95.93169261677549,
|
| 140 |
+
"lr_score": 90.69383655019013,
|
| 141 |
+
"lr_calls": 98.05984071177441,
|
| 142 |
+
"rescued": 2693.8,
|
| 143 |
+
"harmed": 704.8
|
| 144 |
+
},
|
| 145 |
+
{
|
| 146 |
+
"key": [
|
| 147 |
+
"nlvr2",
|
| 148 |
+
"vilt",
|
| 149 |
+
"Qwen3.8"
|
| 150 |
+
],
|
| 151 |
+
"confidence_score": 89.96197172992753,
|
| 152 |
+
"confidence_calls": 95.93169261677549,
|
| 153 |
+
"lr_score": 90.0652938221999,
|
| 154 |
+
"lr_calls": 99.61684724115663,
|
| 155 |
+
"rescued": 2788.2,
|
| 156 |
+
"harmed": 886.8
|
| 157 |
+
},
|
| 158 |
+
{
|
| 159 |
+
"key": [
|
| 160 |
+
"construction",
|
| 161 |
+
"yolo26x",
|
| 162 |
+
"MiniCPM"
|
| 163 |
+
],
|
| 164 |
+
"confidence_score": 25.23047077304935,
|
| 165 |
+
"confidence_calls": 92.54327563249002,
|
| 166 |
+
"lr_score": 25.880436219165098,
|
| 167 |
+
"lr_calls": 45.31957390146471,
|
| 168 |
+
"rescued": 1139.6,
|
| 169 |
+
"harmed": 803.4
|
| 170 |
+
},
|
| 171 |
+
{
|
| 172 |
+
"key": [
|
| 173 |
+
"construction",
|
| 174 |
+
"yolo26x",
|
| 175 |
+
"Qwen3.8"
|
| 176 |
+
],
|
| 177 |
+
"confidence_score": 25.974900047621052,
|
| 178 |
+
"confidence_calls": 92.21038615179761,
|
| 179 |
+
"lr_score": 29.79678330127048,
|
| 180 |
+
"lr_calls": 44.766977363515316,
|
| 181 |
+
"rescued": 1231.6,
|
| 182 |
+
"harmed": 315.0
|
| 183 |
+
},
|
| 184 |
+
{
|
| 185 |
+
"key": [
|
| 186 |
+
"construction",
|
| 187 |
+
"rtdetr_x_fp32",
|
| 188 |
+
"MiniCPM"
|
| 189 |
+
],
|
| 190 |
+
"confidence_score": 22.361115727328066,
|
| 191 |
+
"confidence_calls": 100.0,
|
| 192 |
+
"lr_score": 22.79931433315267,
|
| 193 |
+
"lr_calls": 78.43541944074566,
|
| 194 |
+
"rescued": 3288.6,
|
| 195 |
+
"harmed": 1452.4
|
| 196 |
+
},
|
| 197 |
+
{
|
| 198 |
+
"key": [
|
| 199 |
+
"construction",
|
| 200 |
+
"rtdetr_x_fp32",
|
| 201 |
+
"Qwen3.8"
|
| 202 |
+
],
|
| 203 |
+
"confidence_score": 27.001009067886244,
|
| 204 |
+
"confidence_calls": 100.0,
|
| 205 |
+
"lr_score": 26.783631605767233,
|
| 206 |
+
"lr_calls": 83.56857523302264,
|
| 207 |
+
"rescued": 3448.0,
|
| 208 |
+
"harmed": 574.8
|
| 209 |
+
}
|
| 210 |
+
]
|
releases/20260913_lr/paper_data/main_table_cost_20260912/build_figures.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Main-table curves plus fresh measured component costs; no threshold re-selection."""
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
import csv
|
| 4 |
+
import json
|
| 5 |
+
import numpy as np
|
| 6 |
+
import matplotlib
|
| 7 |
+
matplotlib.use('Agg')
|
| 8 |
+
import matplotlib.pyplot as plt
|
| 9 |
+
from matplotlib.ticker import MaxNLocator,LogLocator,NullLocator
|
| 10 |
+
|
| 11 |
+
root=Path(__file__).resolve().parent
|
| 12 |
+
groups=json.loads((root/'main_table_results.json').read_text(encoding='utf-8'))['groups']
|
| 13 |
+
components={}
|
| 14 |
+
curves=[]
|
| 15 |
+
selected=[]
|
| 16 |
+
for g,tag in zip(groups,['minicpm','qwen']):
|
| 17 |
+
expert=g['key'][1]
|
| 18 |
+
spec=json.loads((root/f'components/{expert}_specialist.json').read_text())
|
| 19 |
+
router=json.loads((root/f'components/{expert}_router.json').read_text())
|
| 20 |
+
large=json.loads((root/f'components/{tag}_latency.json').read_text())
|
| 21 |
+
flops=json.loads((root/f'components/{tag}_flops.json').read_text())
|
| 22 |
+
assert spec['completed'] and router['status']==large['status']==flops['status']=='complete'
|
| 23 |
+
assert router['run']==g['run'] and router['selected_decisions_match_main']
|
| 24 |
+
assert abs(router['actual_call_fraction']-g['methods']['learned']['test_point']['actual_fraction'])<1e-12
|
| 25 |
+
key='|'.join(g['key'])
|
| 26 |
+
c={'specialist_gflops':spec['mean_counted_gflops_per_sample'],
|
| 27 |
+
'router_gflops':router['mean_gflops_per_input'],
|
| 28 |
+
'large_gflops':flops['mean_prefill_gflops_counted']+large['mean_completion_tokens']*flops['mean_decode_one_token_gflops_counted'],
|
| 29 |
+
'specialist_ms':spec['mean_end_to_end_latency_ms_per_sample'],
|
| 30 |
+
'router_ms':router['mean_latency_ms_per_input'],
|
| 31 |
+
'large_ms':large['mean_batch_wall_ms_per_sample'],
|
| 32 |
+
'large_samples':large['measured_count'], 'flops_probes':flops['sample_count'],
|
| 33 |
+
'specialist_samples':spec['sample_count'],'router_samples':router['sample_count'],'batch_size':large['batch_size']}
|
| 34 |
+
components[key]=c
|
| 35 |
+
for method in ['learned','confidence']:
|
| 36 |
+
for kind,points in [('curve',g['methods'][method]['test_curve']),('selected',[g['methods'][method]['test_point']])]:
|
| 37 |
+
for p in points:
|
| 38 |
+
rate=p['actual_fraction']
|
| 39 |
+
row={'pair':key,'method':method,'nominal_validation_budget':p['budget'],'actual_test_call_fraction':rate,
|
| 40 |
+
'test_metric':p['metric'],'estimated_gflops_per_input':c['specialist_gflops']+(c['router_gflops'] if method=='learned' else 0)+rate*c['large_gflops'],
|
| 41 |
+
'estimated_time_ms_per_input':c['specialist_ms']+(c['router_ms'] if method=='learned' else 0)+rate*c['large_ms']}
|
| 42 |
+
(curves if kind=='curve' else selected).append(row)
|
| 43 |
+
for name,rows in [('full_cost_performance_curves',curves),('selected_points',selected),('fixed_budget_cost_performance',[p for p in curves if p['nominal_validation_budget'] in [0,.05,.1,.25,.5,.75,1]])]:
|
| 44 |
+
with (root/(name+'.csv')).open('w',encoding='utf-8',newline='') as f:
|
| 45 |
+
writer=csv.DictWriter(f,fieldnames=list(rows[0]));writer.writeheader();writer.writerows(rows)
|
| 46 |
+
summary={'components':components,'selected_points':selected,'protocol':
|
| 47 |
+
{'method':'original main-table logit contrast; original checkpoints and validation thresholds; seed 42',
|
| 48 |
+
'large_timing':'64 measured examples; batch size one; same full task prompt; two separate warmup examples',
|
| 49 |
+
'cost':'serial component estimates, not a timed integrated service; includes active distilled encoders and routing head only for learned',
|
| 50 |
+
'flops':'multiply-add=2; supported operators; mean prefill (including visual embedding) plus average generated length times one-token decode probe',
|
| 51 |
+
'scope_limits':'mean call cost extrapolated across policies; no policy-specific input/output-length costs; specialist-feature extraction overhead is not separately measured; confidence threshold overhead not separately timed'}}
|
| 52 |
+
(root/'cost_summary.json').write_text(json.dumps(summary,indent=2),encoding='utf-8')
|
| 53 |
+
|
| 54 |
+
plt.rcParams.update({'font.family':'serif','font.serif':['Times New Roman'],'font.size':7,
|
| 55 |
+
'axes.labelsize':7,'xtick.labelsize':6.5,'ytick.labelsize':6.5,'legend.fontsize':7,
|
| 56 |
+
'pdf.fonttype':42,'ps.fonttype':42,'axes.spines.top':False,'axes.spines.right':False,
|
| 57 |
+
'axes.linewidth':.55,'mathtext.fontset':'stix'})
|
| 58 |
+
styles=[('learned','learned','#176f91','-'),('confidence','conf.','#c78324','--')]
|
| 59 |
+
keys=list(components)
|
| 60 |
+
|
| 61 |
+
def panel(ax,key,field):
|
| 62 |
+
for method,label,color,ls in styles:
|
| 63 |
+
pts=[p for p in curves if p['pair']==key and p['method']==method]
|
| 64 |
+
assert len(pts)==101
|
| 65 |
+
x=np.array([p[field] for p in pts]);y=np.array([p['test_metric']*100 for p in pts])
|
| 66 |
+
assert np.all(np.diff(x)>=-1e-7)
|
| 67 |
+
ax.plot(x,y,color=color,ls=ls,lw=1.05,label=label)
|
| 68 |
+
p=next(p for p in selected if p['pair']==key and p['method']==method)
|
| 69 |
+
ax.scatter(p[field],100*p['test_metric'],color=color,s=13,edgecolor='white',linewidth=.4,zorder=5)
|
| 70 |
+
ax.grid(axis='y',color='#dce1e4',lw=.4)
|
| 71 |
+
ax.tick_params(length=2.3,pad=1.5)
|
| 72 |
+
ax.yaxis.set_major_locator(MaxNLocator(nbins=3))
|
| 73 |
+
if 'gflops' in field:
|
| 74 |
+
ax.set_xscale('log');ax.xaxis.set_major_locator(LogLocator(base=10,numticks=4));ax.xaxis.set_minor_locator(NullLocator())
|
| 75 |
+
ax.set_xlabel('Est. GFLOPs/input',labelpad=1)
|
| 76 |
+
else:
|
| 77 |
+
ax.xaxis.set_major_locator(MaxNLocator(nbins=3));ax.set_xlabel('Est. time (ms/input)',labelpad=1)
|
| 78 |
+
ax.set_ylabel('Acc. (%)' if key.startswith('grefcoco') else 'Macro-F1 (%)',labelpad=1)
|
| 79 |
+
|
| 80 |
+
for field,stem in [('estimated_gflops_per_input','flops_performance'),('estimated_time_ms_per_input','latency_performance')]:
|
| 81 |
+
fig,axes=plt.subplots(1,2,figsize=(3.44,1.50))
|
| 82 |
+
fig.subplots_adjust(left=.115,right=.995,bottom=.31,top=.83,wspace=.43)
|
| 83 |
+
for ax,key,title in zip(axes,keys,['gRefCOCO / M','Construction / Q']):
|
| 84 |
+
panel(ax,key,field)
|
| 85 |
+
ax.text(.5,1.10,title,transform=ax.transAxes,ha='center',fontsize=7)
|
| 86 |
+
h,l=axes[0].get_legend_handles_labels()
|
| 87 |
+
fig.legend(h,l,loc='lower center',bbox_to_anchor=(.53,-.025),ncol=2,frameon=False,handlelength=2.2,columnspacing=1.8)
|
| 88 |
+
fig.savefig(root/(stem+'.pdf'));fig.savefig(root/(stem+'.png'),dpi=400)
|
| 89 |
+
plt.close(fig)
|
| 90 |
+
|
| 91 |
+
fig,axes=plt.subplots(2,2,figsize=(3.44,2.05))
|
| 92 |
+
fig.subplots_adjust(left=.115,right=.995,bottom=.20,top=.89,wspace=.43,hspace=.64)
|
| 93 |
+
for col,key in enumerate(keys):
|
| 94 |
+
for row,field in enumerate(['estimated_gflops_per_input','estimated_time_ms_per_input']):
|
| 95 |
+
panel(axes[row,col],key,field)
|
| 96 |
+
axes[0,col].text(.5,1.14,'gRefCOCO / M' if col==0 else 'Construction / Q',transform=axes[0,col].transAxes,ha='center',fontsize=7)
|
| 97 |
+
h,l=axes[0,0].get_legend_handles_labels()
|
| 98 |
+
fig.legend(h,l,loc='lower center',bbox_to_anchor=(.53,-.005),ncol=2,frameon=False,handlelength=2.2,columnspacing=1.8)
|
| 99 |
+
fig.savefig(root/'cost_compact.pdf');fig.savefig(root/'cost_compact.png',dpi=400)
|
| 100 |
+
plt.close(fig)
|
| 101 |
+
print(json.dumps(summary,indent=2))
|
releases/20260913_lr/paper_data/main_table_cost_20260912/components/instancevg_router.json
ADDED
|
@@ -0,0 +1,239 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"run": "shared_12_instancevg_MiniCPM",
|
| 4 |
+
"config": {
|
| 5 |
+
"fraction": 1.0,
|
| 6 |
+
"seed": 42,
|
| 7 |
+
"freeze_encoder": true,
|
| 8 |
+
"fusion": "concat",
|
| 9 |
+
"harm_cost": 1.0,
|
| 10 |
+
"batch_size": 256,
|
| 11 |
+
"weight_decay": 0.01,
|
| 12 |
+
"max_epochs": 50,
|
| 13 |
+
"patience": 8,
|
| 14 |
+
"cpu_threads": 1,
|
| 15 |
+
"student": "8M",
|
| 16 |
+
"representation_dim": 128,
|
| 17 |
+
"lr": 0.003,
|
| 18 |
+
"target": "four_state",
|
| 19 |
+
"strategy": "natural",
|
| 20 |
+
"hidden_percent": 90,
|
| 21 |
+
"expert": "instancevg",
|
| 22 |
+
"task": "grefcoco",
|
| 23 |
+
"endpoint": "MiniCPM",
|
| 24 |
+
"branches": [
|
| 25 |
+
"image",
|
| 26 |
+
"text",
|
| 27 |
+
"hidden",
|
| 28 |
+
"confidence",
|
| 29 |
+
"output"
|
| 30 |
+
],
|
| 31 |
+
"hidden_layer": "joint_encoder__layer_11",
|
| 32 |
+
"id": "shared_12_instancevg_MiniCPM",
|
| 33 |
+
"stage": "shared_mandatory_inputs"
|
| 34 |
+
},
|
| 35 |
+
"checkpoint": "/mnt/HithinkOmniSSD/user_workspace/caisihang/research/ICASSP_ModelCollaboration/outputs/router_shared_image_hidden_20260910/runs/shared_12_instancevg_MiniCPM/best.pt",
|
| 36 |
+
"score": "logit_rescue-logit_harm",
|
| 37 |
+
"test_rows_recomputed": 35263,
|
| 38 |
+
"selected_decisions_match_main": true,
|
| 39 |
+
"max_score_abs_difference": 0.0,
|
| 40 |
+
"actual_call_fraction": 0.30904914499617164,
|
| 41 |
+
"sample_count": 64,
|
| 42 |
+
"sample_ids": [
|
| 43 |
+
"testB-5056-14012",
|
| 44 |
+
"testB-31505-86708",
|
| 45 |
+
"testA-57489-158264",
|
| 46 |
+
"testA-38865-107108",
|
| 47 |
+
"testB-47361-130361",
|
| 48 |
+
"testA-5476-15123",
|
| 49 |
+
"testA-71404-196892",
|
| 50 |
+
"testA-61030-168115",
|
| 51 |
+
"testB-35969-99063",
|
| 52 |
+
"testB-1675-4724",
|
| 53 |
+
"testA-30414-83641",
|
| 54 |
+
"testB-47742-131425",
|
| 55 |
+
"testB-77456-213280",
|
| 56 |
+
"testA-59283-163253",
|
| 57 |
+
"testA-57390-157996",
|
| 58 |
+
"testA-77229-212665",
|
| 59 |
+
"testA-6784-18703",
|
| 60 |
+
"testB-11434-31542",
|
| 61 |
+
"testB-11675-32181",
|
| 62 |
+
"testB-73046-201304",
|
| 63 |
+
"testA-28837-79356",
|
| 64 |
+
"testA-4563-12619",
|
| 65 |
+
"testB-30168-82996",
|
| 66 |
+
"testB-92461-254721",
|
| 67 |
+
"testA-58293-160462",
|
| 68 |
+
"testA-13050-35910",
|
| 69 |
+
"testA-39320-108309",
|
| 70 |
+
"testB-36423-100313",
|
| 71 |
+
"testA-43418-119528",
|
| 72 |
+
"testB-23411-64351",
|
| 73 |
+
"testB-77730-214031",
|
| 74 |
+
"testA-7232-19907",
|
| 75 |
+
"testA-17738-48695",
|
| 76 |
+
"testA-44914-123593",
|
| 77 |
+
"testB-92861-255854",
|
| 78 |
+
"testA-69678-192132",
|
| 79 |
+
"testA-68785-189658",
|
| 80 |
+
"testB-62911-173380",
|
| 81 |
+
"testB-19029-52242",
|
| 82 |
+
"testA-66284-182759",
|
| 83 |
+
"testB-19170-52626",
|
| 84 |
+
"testB-8468-23291",
|
| 85 |
+
"testB-72940-201009",
|
| 86 |
+
"testB-74045-203983",
|
| 87 |
+
"testB-93380-257275",
|
| 88 |
+
"testB-49369-135990",
|
| 89 |
+
"testA-58046-159759",
|
| 90 |
+
"testA-57437-158132",
|
| 91 |
+
"testA-5754-15895",
|
| 92 |
+
"testA-85590-235577",
|
| 93 |
+
"testA-92278-254227",
|
| 94 |
+
"testB-80373-221257",
|
| 95 |
+
"testA-80383-221294",
|
| 96 |
+
"testB-71197-196311",
|
| 97 |
+
"testB-48080-132352",
|
| 98 |
+
"testB-18351-50380",
|
| 99 |
+
"testB-1089-3077",
|
| 100 |
+
"testB-92466-254737",
|
| 101 |
+
"testB-53065-146099",
|
| 102 |
+
"testB-42611-117349",
|
| 103 |
+
"testA-57690-158796",
|
| 104 |
+
"testA-65173-179681",
|
| 105 |
+
"testB-90675-249707",
|
| 106 |
+
"testB-74709-205815"
|
| 107 |
+
],
|
| 108 |
+
"batch_size": 1,
|
| 109 |
+
"latency_ms": [
|
| 110 |
+
13.662165962159634,
|
| 111 |
+
9.513642988167703,
|
| 112 |
+
8.86400998570025,
|
| 113 |
+
7.89595500100404,
|
| 114 |
+
9.713676990941167,
|
| 115 |
+
9.162381989881396,
|
| 116 |
+
9.548410074785352,
|
| 117 |
+
9.09417902585119,
|
| 118 |
+
11.833172990009189,
|
| 119 |
+
14.292752020992339,
|
| 120 |
+
10.746326064690948,
|
| 121 |
+
10.920883971266448,
|
| 122 |
+
19.845983013510704,
|
| 123 |
+
14.003381016664207,
|
| 124 |
+
15.885940054431558,
|
| 125 |
+
14.8980199592188,
|
| 126 |
+
15.25757904164493,
|
| 127 |
+
17.458743997849524,
|
| 128 |
+
17.598357051610947,
|
| 129 |
+
15.274635050445795,
|
| 130 |
+
15.84710506722331,
|
| 131 |
+
31.497937976382673,
|
| 132 |
+
17.706500948406756,
|
| 133 |
+
13.621860067360103,
|
| 134 |
+
12.55292899440974,
|
| 135 |
+
10.675732046365738,
|
| 136 |
+
9.522471926175058,
|
| 137 |
+
11.19849900715053,
|
| 138 |
+
13.095670961774886,
|
| 139 |
+
10.178428026847541,
|
| 140 |
+
11.139138019643724,
|
| 141 |
+
10.480726952664554,
|
| 142 |
+
9.46063396986574,
|
| 143 |
+
10.675741010345519,
|
| 144 |
+
9.826205903664231,
|
| 145 |
+
9.403666015714407,
|
| 146 |
+
10.096908081322908,
|
| 147 |
+
28.78253604285419,
|
| 148 |
+
21.029149065725505,
|
| 149 |
+
10.848066071048379,
|
| 150 |
+
10.954314027912915,
|
| 151 |
+
11.451196041889489,
|
| 152 |
+
11.626868974417448,
|
| 153 |
+
9.91171004716307,
|
| 154 |
+
12.449359055608511,
|
| 155 |
+
11.346090934239328,
|
| 156 |
+
10.699236067011952,
|
| 157 |
+
10.140218073502183,
|
| 158 |
+
12.354035046882927,
|
| 159 |
+
14.74389701616019,
|
| 160 |
+
11.097895912826061,
|
| 161 |
+
11.268700007349253,
|
| 162 |
+
12.365077971480787,
|
| 163 |
+
10.821785079315305,
|
| 164 |
+
11.905013001523912,
|
| 165 |
+
25.439211982302368,
|
| 166 |
+
15.189048950560391,
|
| 167 |
+
12.554107001051307,
|
| 168 |
+
10.584128089249134,
|
| 169 |
+
23.863457958213985,
|
| 170 |
+
14.576193992979825,
|
| 171 |
+
11.179295019246638,
|
| 172 |
+
12.35828804783523,
|
| 173 |
+
11.567718931473792
|
| 174 |
+
],
|
| 175 |
+
"mean_latency_ms_per_input": 13.180577338061994,
|
| 176 |
+
"head_mean_latency_ms": 0.6150530008599162,
|
| 177 |
+
"gflops_per_input": [
|
| 178 |
+
3.642171392,
|
| 179 |
+
3.642171392,
|
| 180 |
+
3.642171392,
|
| 181 |
+
3.642171392,
|
| 182 |
+
3.642171392,
|
| 183 |
+
3.642171392,
|
| 184 |
+
3.642171392,
|
| 185 |
+
3.642171392
|
| 186 |
+
],
|
| 187 |
+
"mean_gflops_per_input": 3.642171392,
|
| 188 |
+
"operators": [
|
| 189 |
+
{
|
| 190 |
+
"aten.convolution": 228802560,
|
| 191 |
+
"aten.bmm": 691709440,
|
| 192 |
+
"aten._scaled_dot_product_flash_attention": 236027904,
|
| 193 |
+
"aten.addmm": 2485631488
|
| 194 |
+
},
|
| 195 |
+
{
|
| 196 |
+
"aten.convolution": 228802560,
|
| 197 |
+
"aten.bmm": 691709440,
|
| 198 |
+
"aten._scaled_dot_product_flash_attention": 236027904,
|
| 199 |
+
"aten.addmm": 2485631488
|
| 200 |
+
},
|
| 201 |
+
{
|
| 202 |
+
"aten.convolution": 228802560,
|
| 203 |
+
"aten.bmm": 691709440,
|
| 204 |
+
"aten._scaled_dot_product_flash_attention": 236027904,
|
| 205 |
+
"aten.addmm": 2485631488
|
| 206 |
+
},
|
| 207 |
+
{
|
| 208 |
+
"aten.convolution": 228802560,
|
| 209 |
+
"aten.bmm": 691709440,
|
| 210 |
+
"aten._scaled_dot_product_flash_attention": 236027904,
|
| 211 |
+
"aten.addmm": 2485631488
|
| 212 |
+
},
|
| 213 |
+
{
|
| 214 |
+
"aten.convolution": 228802560,
|
| 215 |
+
"aten.bmm": 691709440,
|
| 216 |
+
"aten._scaled_dot_product_flash_attention": 236027904,
|
| 217 |
+
"aten.addmm": 2485631488
|
| 218 |
+
},
|
| 219 |
+
{
|
| 220 |
+
"aten.convolution": 228802560,
|
| 221 |
+
"aten.bmm": 691709440,
|
| 222 |
+
"aten._scaled_dot_product_flash_attention": 236027904,
|
| 223 |
+
"aten.addmm": 2485631488
|
| 224 |
+
},
|
| 225 |
+
{
|
| 226 |
+
"aten.convolution": 228802560,
|
| 227 |
+
"aten.bmm": 691709440,
|
| 228 |
+
"aten._scaled_dot_product_flash_attention": 236027904,
|
| 229 |
+
"aten.addmm": 2485631488
|
| 230 |
+
},
|
| 231 |
+
{
|
| 232 |
+
"aten.convolution": 228802560,
|
| 233 |
+
"aten.bmm": 691709440,
|
| 234 |
+
"aten._scaled_dot_product_flash_attention": 236027904,
|
| 235 |
+
"aten.addmm": 2485631488
|
| 236 |
+
}
|
| 237 |
+
],
|
| 238 |
+
"scope": "image loading, bilinear resize, tokenize when used, two image slots (including gray placeholder), active distilled encoders, feature assembly and original routing head; cached specialist features; supported FLOPs only"
|
| 239 |
+
}
|
releases/20260913_lr/paper_data/main_table_cost_20260912/components/instancevg_specialist.json
ADDED
|
@@ -0,0 +1,254 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"component": "specialist",
|
| 3 |
+
"expert": "instancevg",
|
| 4 |
+
"task": "grefcoco",
|
| 5 |
+
"device": "cuda:0",
|
| 6 |
+
"checkpoint": "/mnt/HithinkOmniSSD/user_workspace/caisihang/research/ICASSP_ModelCollaboration/models/grefcoco/instancevg/InstanceVG-grefcoco.pth",
|
| 7 |
+
"sample_count": 64,
|
| 8 |
+
"batch_size": 1,
|
| 9 |
+
"sample_ids": [
|
| 10 |
+
"testA-30844-84861",
|
| 11 |
+
"testB-44353-122077",
|
| 12 |
+
"testB-36889-101643",
|
| 13 |
+
"testA-49055-135087",
|
| 14 |
+
"testA-81245-223633",
|
| 15 |
+
"testB-37669-103833",
|
| 16 |
+
"testA-5038-13943",
|
| 17 |
+
"testB-79646-219274",
|
| 18 |
+
"testA-469-1309",
|
| 19 |
+
"testB-46644-128438",
|
| 20 |
+
"testA-24562-67515",
|
| 21 |
+
"testA-47070-129599",
|
| 22 |
+
"testA-54554-150141",
|
| 23 |
+
"testB-12382-34099",
|
| 24 |
+
"testB-23014-63279",
|
| 25 |
+
"testB-79647-219275",
|
| 26 |
+
"testB-69479-191564",
|
| 27 |
+
"testA-15582-42797",
|
| 28 |
+
"testB-56692-156000",
|
| 29 |
+
"testB-42726-117660",
|
| 30 |
+
"testA-80455-221494",
|
| 31 |
+
"testB-33115-91174",
|
| 32 |
+
"testA-62940-173455",
|
| 33 |
+
"testB-31494-86670",
|
| 34 |
+
"testA-54586-150233",
|
| 35 |
+
"testA-81445-224177",
|
| 36 |
+
"testB-8417-23141",
|
| 37 |
+
"testA-24438-67190",
|
| 38 |
+
"testA-59285-163255",
|
| 39 |
+
"testA-73258-201893",
|
| 40 |
+
"testB-945-2633",
|
| 41 |
+
"testA-54307-149426",
|
| 42 |
+
"testB-60557-166822",
|
| 43 |
+
"testA-81622-224665",
|
| 44 |
+
"testA-69466-191525",
|
| 45 |
+
"testA-60731-167321",
|
| 46 |
+
"testA-7857-21644",
|
| 47 |
+
"testA-44142-121499",
|
| 48 |
+
"testB-66162-182407",
|
| 49 |
+
"testA-31764-87406",
|
| 50 |
+
"testB-38637-106482",
|
| 51 |
+
"testB-52921-145709",
|
| 52 |
+
"testB-65796-181394",
|
| 53 |
+
"testB-2936-8113",
|
| 54 |
+
"testA-79438-218690",
|
| 55 |
+
"testA-57397-158016",
|
| 56 |
+
"testB-23920-65743",
|
| 57 |
+
"testB-15419-42358",
|
| 58 |
+
"testA-10078-27739",
|
| 59 |
+
"testA-43890-120793",
|
| 60 |
+
"testA-55116-151689",
|
| 61 |
+
"testB-27487-75582",
|
| 62 |
+
"testA-26856-73830",
|
| 63 |
+
"testB-50845-140107",
|
| 64 |
+
"testA-29199-80374",
|
| 65 |
+
"testB-65935-181778",
|
| 66 |
+
"testA-17736-48684",
|
| 67 |
+
"testB-69477-191553",
|
| 68 |
+
"testB-22290-61227",
|
| 69 |
+
"testA-58476-160950",
|
| 70 |
+
"testB-36265-99859",
|
| 71 |
+
"testA-5734-15837",
|
| 72 |
+
"testB-67952-187353",
|
| 73 |
+
"testB-47054-129555"
|
| 74 |
+
],
|
| 75 |
+
"model_load_seconds_excluded_from_latency": true,
|
| 76 |
+
"forward_latency_ms_per_sample": [
|
| 77 |
+
41.24195098876953,
|
| 78 |
+
41.74086380004883,
|
| 79 |
+
40.36540985107422,
|
| 80 |
+
44.46102523803711,
|
| 81 |
+
41.2421760559082,
|
| 82 |
+
40.76591873168945,
|
| 83 |
+
41.14704132080078,
|
| 84 |
+
40.88172912597656,
|
| 85 |
+
41.70716857910156,
|
| 86 |
+
41.191871643066406,
|
| 87 |
+
39.91231918334961,
|
| 88 |
+
39.29497528076172,
|
| 89 |
+
40.70723342895508,
|
| 90 |
+
38.920894622802734,
|
| 91 |
+
39.32780838012695,
|
| 92 |
+
38.89958572387695,
|
| 93 |
+
38.59721755981445,
|
| 94 |
+
38.47331237792969,
|
| 95 |
+
39.8744010925293,
|
| 96 |
+
39.87260818481445,
|
| 97 |
+
41.48284912109375,
|
| 98 |
+
40.55779266357422,
|
| 99 |
+
39.05420684814453,
|
| 100 |
+
39.052734375,
|
| 101 |
+
39.143775939941406,
|
| 102 |
+
39.82249450683594,
|
| 103 |
+
39.543006896972656,
|
| 104 |
+
39.368160247802734,
|
| 105 |
+
40.661216735839844,
|
| 106 |
+
39.092193603515625,
|
| 107 |
+
39.85852813720703,
|
| 108 |
+
40.33164978027344,
|
| 109 |
+
38.89593505859375,
|
| 110 |
+
39.29216003417969,
|
| 111 |
+
39.20943832397461,
|
| 112 |
+
40.946590423583984,
|
| 113 |
+
39.44643020629883,
|
| 114 |
+
40.220863342285156,
|
| 115 |
+
39.61030578613281,
|
| 116 |
+
39.639007568359375,
|
| 117 |
+
39.78163146972656,
|
| 118 |
+
39.223201751708984,
|
| 119 |
+
39.00921630859375,
|
| 120 |
+
38.99894332885742,
|
| 121 |
+
41.13721466064453,
|
| 122 |
+
39.94972610473633,
|
| 123 |
+
41.41766357421875,
|
| 124 |
+
39.28073501586914,
|
| 125 |
+
39.721214294433594,
|
| 126 |
+
39.24937438964844,
|
| 127 |
+
39.40460968017578,
|
| 128 |
+
41.30179214477539,
|
| 129 |
+
42.94054412841797,
|
| 130 |
+
41.5063362121582,
|
| 131 |
+
44.03267288208008,
|
| 132 |
+
44.55561447143555,
|
| 133 |
+
44.24995040893555,
|
| 134 |
+
44.87868881225586,
|
| 135 |
+
45.14652633666992,
|
| 136 |
+
44.85055923461914,
|
| 137 |
+
42.39228820800781,
|
| 138 |
+
41.121856689453125,
|
| 139 |
+
45.708351135253906,
|
| 140 |
+
41.8752326965332
|
| 141 |
+
],
|
| 142 |
+
"end_to_end_latency_ms_per_sample": [
|
| 143 |
+
47.29630902875215,
|
| 144 |
+
46.27182998228818,
|
| 145 |
+
47.56885801907629,
|
| 146 |
+
45.48592597711831,
|
| 147 |
+
45.399176073260605,
|
| 148 |
+
46.442538034170866,
|
| 149 |
+
45.95420998521149,
|
| 150 |
+
46.45739798434079,
|
| 151 |
+
44.90085388533771,
|
| 152 |
+
43.80481690168381,
|
| 153 |
+
44.848884106613696,
|
| 154 |
+
43.67590101901442,
|
| 155 |
+
46.40063806436956,
|
| 156 |
+
44.26892101764679,
|
| 157 |
+
44.44277100265026,
|
| 158 |
+
44.71806308720261,
|
| 159 |
+
44.62323593907058,
|
| 160 |
+
44.6264409692958,
|
| 161 |
+
44.96095096692443,
|
| 162 |
+
44.103492982685566,
|
| 163 |
+
45.13338196557015,
|
| 164 |
+
44.1846780013293,
|
| 165 |
+
48.1606520479545,
|
| 166 |
+
46.43543600104749,
|
| 167 |
+
46.19023704435676,
|
| 168 |
+
45.247678994201124,
|
| 169 |
+
44.801001087762415,
|
| 170 |
+
45.467636082321405,
|
| 171 |
+
44.22409099061042,
|
| 172 |
+
44.7448689956218,
|
| 173 |
+
58.0457029864192,
|
| 174 |
+
46.10048606991768,
|
| 175 |
+
44.56539999227971,
|
| 176 |
+
44.47658592835069,
|
| 177 |
+
46.05250502936542,
|
| 178 |
+
44.743441976606846,
|
| 179 |
+
44.77924096863717,
|
| 180 |
+
43.877715012058616,
|
| 181 |
+
44.05005706939846,
|
| 182 |
+
42.87725198082626,
|
| 183 |
+
43.836933095008135,
|
| 184 |
+
45.634721987880766,
|
| 185 |
+
44.748114072717726,
|
| 186 |
+
46.21323198080063,
|
| 187 |
+
52.24333005025983,
|
| 188 |
+
48.90036000870168,
|
| 189 |
+
48.12136595137417,
|
| 190 |
+
47.65231907367706,
|
| 191 |
+
46.71858798246831,
|
| 192 |
+
48.516221111640334,
|
| 193 |
+
43.90157596208155,
|
| 194 |
+
46.83465394191444,
|
| 195 |
+
46.49809293914586,
|
| 196 |
+
42.6300170365721,
|
| 197 |
+
47.39421408157796,
|
| 198 |
+
48.35387098137289,
|
| 199 |
+
48.176085110753775,
|
| 200 |
+
48.29539393540472,
|
| 201 |
+
47.32040502130985,
|
| 202 |
+
46.35261802468449,
|
| 203 |
+
47.482443042099476,
|
| 204 |
+
47.437409055419266,
|
| 205 |
+
47.27676196489483,
|
| 206 |
+
48.198972013778985
|
| 207 |
+
],
|
| 208 |
+
"mean_forward_latency_ms_per_sample": 40.71232491731644,
|
| 209 |
+
"median_forward_latency_ms_per_sample": 40.08529472351074,
|
| 210 |
+
"mean_end_to_end_latency_ms_per_sample": 46.08042127620138,
|
| 211 |
+
"median_end_to_end_latency_ms_per_sample": 45.79446598654613,
|
| 212 |
+
"flop_profiles": [
|
| 213 |
+
{
|
| 214 |
+
"batch_size": 1,
|
| 215 |
+
"flops_counted": 7502680064,
|
| 216 |
+
"gflops_counted_per_sample": 7.502680064,
|
| 217 |
+
"operators": {
|
| 218 |
+
"aten.bmm": 7500632064,
|
| 219 |
+
"aten.baddbmm": 2048000
|
| 220 |
+
}
|
| 221 |
+
},
|
| 222 |
+
{
|
| 223 |
+
"batch_size": 1,
|
| 224 |
+
"flops_counted": 7502680064,
|
| 225 |
+
"gflops_counted_per_sample": 7.502680064,
|
| 226 |
+
"operators": {
|
| 227 |
+
"aten.bmm": 7500632064,
|
| 228 |
+
"aten.baddbmm": 2048000
|
| 229 |
+
}
|
| 230 |
+
},
|
| 231 |
+
{
|
| 232 |
+
"batch_size": 1,
|
| 233 |
+
"flops_counted": 7502680064,
|
| 234 |
+
"gflops_counted_per_sample": 7.502680064,
|
| 235 |
+
"operators": {
|
| 236 |
+
"aten.bmm": 7500632064,
|
| 237 |
+
"aten.baddbmm": 2048000
|
| 238 |
+
}
|
| 239 |
+
},
|
| 240 |
+
{
|
| 241 |
+
"batch_size": 1,
|
| 242 |
+
"flops_counted": 7502680064,
|
| 243 |
+
"gflops_counted_per_sample": 7.502680064,
|
| 244 |
+
"operators": {
|
| 245 |
+
"aten.bmm": 7500632064,
|
| 246 |
+
"aten.baddbmm": 2048000
|
| 247 |
+
}
|
| 248 |
+
}
|
| 249 |
+
],
|
| 250 |
+
"mean_counted_gflops_per_sample": 7.502680064,
|
| 251 |
+
"flops_scope": "GPU FlopCounterMode dense products, convolutions and supported attention; multiply-add counted as 2 FLOPs. Elementwise, normalization, custom ops and postprocessing may be missing.",
|
| 252 |
+
"latency_scope": "A800 wall-clock CUDA forward latency and end-to-end latency including image read/preprocess, excluding model load; batch processing uses the recorded batch size.",
|
| 253 |
+
"completed": true
|
| 254 |
+
}
|
releases/20260913_lr/paper_data/main_table_cost_20260912/components/minicpm_flops.json
ADDED
|
@@ -0,0 +1,261 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"model_kind": "MiniCPM",
|
| 4 |
+
"task": "grefcoco",
|
| 5 |
+
"model_path": "/mnt/HithinkOmniSSD/user_workspace/caisihang/checkpoints/MiniCPM-V-4_5",
|
| 6 |
+
"sample_count": 8,
|
| 7 |
+
"records": [
|
| 8 |
+
{
|
| 9 |
+
"sample_id": "testA-5038-13945",
|
| 10 |
+
"prompt_tokens_in_reference_tensor": 528,
|
| 11 |
+
"input_shapes": {
|
| 12 |
+
"input_ids": [
|
| 13 |
+
1,
|
| 14 |
+
528
|
| 15 |
+
],
|
| 16 |
+
"attention_mask": [
|
| 17 |
+
1,
|
| 18 |
+
528
|
| 19 |
+
]
|
| 20 |
+
},
|
| 21 |
+
"prefill_flops_counted": 10720928684032,
|
| 22 |
+
"decode_one_token_flops_counted": 15446671488,
|
| 23 |
+
"prefill_gflops_counted": 10720.928684032,
|
| 24 |
+
"decode_one_token_gflops_counted": 15.446671488,
|
| 25 |
+
"prefill_operators": {
|
| 26 |
+
"aten.bmm": 164433561600,
|
| 27 |
+
"aten.mm": 7335973519360,
|
| 28 |
+
"vision:aten.convolution": 4210569216,
|
| 29 |
+
"vision:aten.addmm": 2568098906112,
|
| 30 |
+
"vision:aten.bmm": 610809421824,
|
| 31 |
+
"vision:aten.mm": 35773218816,
|
| 32 |
+
"vision:aten.baddbmm": 1629487104
|
| 33 |
+
},
|
| 34 |
+
"decode_operators": {
|
| 35 |
+
"aten.bmm": 312017024,
|
| 36 |
+
"aten.mm": 15134654464
|
| 37 |
+
}
|
| 38 |
+
},
|
| 39 |
+
{
|
| 40 |
+
"sample_id": "testA-73305-202014",
|
| 41 |
+
"prompt_tokens_in_reference_tensor": 528,
|
| 42 |
+
"input_shapes": {
|
| 43 |
+
"input_ids": [
|
| 44 |
+
1,
|
| 45 |
+
528
|
| 46 |
+
],
|
| 47 |
+
"attention_mask": [
|
| 48 |
+
1,
|
| 49 |
+
528
|
| 50 |
+
]
|
| 51 |
+
},
|
| 52 |
+
"prefill_flops_counted": 10720928684032,
|
| 53 |
+
"decode_one_token_flops_counted": 15446671488,
|
| 54 |
+
"prefill_gflops_counted": 10720.928684032,
|
| 55 |
+
"decode_one_token_gflops_counted": 15.446671488,
|
| 56 |
+
"prefill_operators": {
|
| 57 |
+
"aten.bmm": 164433561600,
|
| 58 |
+
"aten.mm": 7335973519360,
|
| 59 |
+
"vision:aten.convolution": 4210569216,
|
| 60 |
+
"vision:aten.addmm": 2568098906112,
|
| 61 |
+
"vision:aten.bmm": 610809421824,
|
| 62 |
+
"vision:aten.mm": 35773218816,
|
| 63 |
+
"vision:aten.baddbmm": 1629487104
|
| 64 |
+
},
|
| 65 |
+
"decode_operators": {
|
| 66 |
+
"aten.bmm": 312017024,
|
| 67 |
+
"aten.mm": 15134654464
|
| 68 |
+
}
|
| 69 |
+
},
|
| 70 |
+
{
|
| 71 |
+
"sample_id": "testA-26952-74099",
|
| 72 |
+
"prompt_tokens_in_reference_tensor": 523,
|
| 73 |
+
"input_shapes": {
|
| 74 |
+
"input_ids": [
|
| 75 |
+
1,
|
| 76 |
+
523
|
| 77 |
+
],
|
| 78 |
+
"attention_mask": [
|
| 79 |
+
1,
|
| 80 |
+
523
|
| 81 |
+
]
|
| 82 |
+
},
|
| 83 |
+
"prefill_flops_counted": 10648371484032,
|
| 84 |
+
"decode_one_token_flops_counted": 15443722368,
|
| 85 |
+
"prefill_gflops_counted": 10648.371484032,
|
| 86 |
+
"decode_one_token_gflops_counted": 15.443722368,
|
| 87 |
+
"prefill_operators": {
|
| 88 |
+
"aten.bmm": 161334035840,
|
| 89 |
+
"aten.mm": 7266515845120,
|
| 90 |
+
"vision:aten.convolution": 4210569216,
|
| 91 |
+
"vision:aten.addmm": 2568098906112,
|
| 92 |
+
"vision:aten.bmm": 610809421824,
|
| 93 |
+
"vision:aten.mm": 35773218816,
|
| 94 |
+
"vision:aten.baddbmm": 1629487104
|
| 95 |
+
},
|
| 96 |
+
"decode_operators": {
|
| 97 |
+
"aten.bmm": 309067904,
|
| 98 |
+
"aten.mm": 15134654464
|
| 99 |
+
}
|
| 100 |
+
},
|
| 101 |
+
{
|
| 102 |
+
"sample_id": "testB-53254-146620",
|
| 103 |
+
"prompt_tokens_in_reference_tensor": 520,
|
| 104 |
+
"input_shapes": {
|
| 105 |
+
"input_ids": [
|
| 106 |
+
1,
|
| 107 |
+
520
|
| 108 |
+
],
|
| 109 |
+
"attention_mask": [
|
| 110 |
+
1,
|
| 111 |
+
520
|
| 112 |
+
]
|
| 113 |
+
},
|
| 114 |
+
"prefill_flops_counted": 10604851319808,
|
| 115 |
+
"decode_one_token_flops_counted": 15441952896,
|
| 116 |
+
"prefill_gflops_counted": 10604.851319808,
|
| 117 |
+
"decode_one_token_gflops_counted": 15.441952896,
|
| 118 |
+
"prefill_operators": {
|
| 119 |
+
"aten.bmm": 159488476160,
|
| 120 |
+
"aten.mm": 7224841240576,
|
| 121 |
+
"vision:aten.convolution": 4210569216,
|
| 122 |
+
"vision:aten.addmm": 2568098906112,
|
| 123 |
+
"vision:aten.bmm": 610809421824,
|
| 124 |
+
"vision:aten.mm": 35773218816,
|
| 125 |
+
"vision:aten.baddbmm": 1629487104
|
| 126 |
+
},
|
| 127 |
+
"decode_operators": {
|
| 128 |
+
"aten.bmm": 307298432,
|
| 129 |
+
"aten.mm": 15134654464
|
| 130 |
+
}
|
| 131 |
+
},
|
| 132 |
+
{
|
| 133 |
+
"sample_id": "testB-37889-104439",
|
| 134 |
+
"prompt_tokens_in_reference_tensor": 528,
|
| 135 |
+
"input_shapes": {
|
| 136 |
+
"input_ids": [
|
| 137 |
+
1,
|
| 138 |
+
528
|
| 139 |
+
],
|
| 140 |
+
"attention_mask": [
|
| 141 |
+
1,
|
| 142 |
+
528
|
| 143 |
+
]
|
| 144 |
+
},
|
| 145 |
+
"prefill_flops_counted": 10637704296448,
|
| 146 |
+
"decode_one_token_flops_counted": 15446671488,
|
| 147 |
+
"prefill_gflops_counted": 10637.704296448,
|
| 148 |
+
"decode_one_token_gflops_counted": 15.446671488,
|
| 149 |
+
"prefill_operators": {
|
| 150 |
+
"aten.bmm": 164433561600,
|
| 151 |
+
"aten.mm": 7335973519360,
|
| 152 |
+
"vision:aten.convolution": 4113027072,
|
| 153 |
+
"vision:aten.addmm": 2508904759296,
|
| 154 |
+
"vision:aten.bmm": 587593949184,
|
| 155 |
+
"vision:aten.mm": 35093741568,
|
| 156 |
+
"vision:aten.baddbmm": 1591738368
|
| 157 |
+
},
|
| 158 |
+
"decode_operators": {
|
| 159 |
+
"aten.bmm": 312017024,
|
| 160 |
+
"aten.mm": 15134654464
|
| 161 |
+
}
|
| 162 |
+
},
|
| 163 |
+
{
|
| 164 |
+
"sample_id": "testA-58523-161084",
|
| 165 |
+
"prompt_tokens_in_reference_tensor": 396,
|
| 166 |
+
"input_shapes": {
|
| 167 |
+
"input_ids": [
|
| 168 |
+
1,
|
| 169 |
+
396
|
| 170 |
+
],
|
| 171 |
+
"attention_mask": [
|
| 172 |
+
1,
|
| 173 |
+
396
|
| 174 |
+
]
|
| 175 |
+
},
|
| 176 |
+
"prefill_flops_counted": 6668292011520,
|
| 177 |
+
"decode_one_token_flops_counted": 15368814720,
|
| 178 |
+
"prefill_gflops_counted": 6668.29201152,
|
| 179 |
+
"decode_one_token_gflops_counted": 15.36881472,
|
| 180 |
+
"prefill_operators": {
|
| 181 |
+
"aten.bmm": 92493891072,
|
| 182 |
+
"aten.mm": 5502290919424,
|
| 183 |
+
"vision:aten.convolution": 1403523072,
|
| 184 |
+
"vision:aten.addmm": 925557751808,
|
| 185 |
+
"vision:aten.bmm": 134078357504,
|
| 186 |
+
"vision:aten.mm": 11924406272,
|
| 187 |
+
"vision:aten.baddbmm": 543162368
|
| 188 |
+
},
|
| 189 |
+
"decode_operators": {
|
| 190 |
+
"aten.bmm": 234160256,
|
| 191 |
+
"aten.mm": 15134654464
|
| 192 |
+
}
|
| 193 |
+
},
|
| 194 |
+
{
|
| 195 |
+
"sample_id": "testB-3419-9449",
|
| 196 |
+
"prompt_tokens_in_reference_tensor": 528,
|
| 197 |
+
"input_shapes": {
|
| 198 |
+
"input_ids": [
|
| 199 |
+
1,
|
| 200 |
+
528
|
| 201 |
+
],
|
| 202 |
+
"attention_mask": [
|
| 203 |
+
1,
|
| 204 |
+
528
|
| 205 |
+
]
|
| 206 |
+
},
|
| 207 |
+
"prefill_flops_counted": 10720928684032,
|
| 208 |
+
"decode_one_token_flops_counted": 15446671488,
|
| 209 |
+
"prefill_gflops_counted": 10720.928684032,
|
| 210 |
+
"decode_one_token_gflops_counted": 15.446671488,
|
| 211 |
+
"prefill_operators": {
|
| 212 |
+
"aten.bmm": 164433561600,
|
| 213 |
+
"aten.mm": 7335973519360,
|
| 214 |
+
"vision:aten.convolution": 4210569216,
|
| 215 |
+
"vision:aten.addmm": 2568098906112,
|
| 216 |
+
"vision:aten.bmm": 610809421824,
|
| 217 |
+
"vision:aten.mm": 35773218816,
|
| 218 |
+
"vision:aten.baddbmm": 1629487104
|
| 219 |
+
},
|
| 220 |
+
"decode_operators": {
|
| 221 |
+
"aten.bmm": 312017024,
|
| 222 |
+
"aten.mm": 15134654464
|
| 223 |
+
}
|
| 224 |
+
},
|
| 225 |
+
{
|
| 226 |
+
"sample_id": "testB-24035-66084",
|
| 227 |
+
"prompt_tokens_in_reference_tensor": 521,
|
| 228 |
+
"input_shapes": {
|
| 229 |
+
"input_ids": [
|
| 230 |
+
1,
|
| 231 |
+
521
|
| 232 |
+
],
|
| 233 |
+
"attention_mask": [
|
| 234 |
+
1,
|
| 235 |
+
521
|
| 236 |
+
]
|
| 237 |
+
},
|
| 238 |
+
"prefill_flops_counted": 10619356861568,
|
| 239 |
+
"decode_one_token_flops_counted": 15442542720,
|
| 240 |
+
"prefill_gflops_counted": 10619.356861568,
|
| 241 |
+
"decode_one_token_gflops_counted": 15.44254272,
|
| 242 |
+
"prefill_operators": {
|
| 243 |
+
"aten.bmm": 160102483072,
|
| 244 |
+
"aten.mm": 7238732775424,
|
| 245 |
+
"vision:aten.convolution": 4210569216,
|
| 246 |
+
"vision:aten.addmm": 2568098906112,
|
| 247 |
+
"vision:aten.bmm": 610809421824,
|
| 248 |
+
"vision:aten.mm": 35773218816,
|
| 249 |
+
"vision:aten.baddbmm": 1629487104
|
| 250 |
+
},
|
| 251 |
+
"decode_operators": {
|
| 252 |
+
"aten.bmm": 307888256,
|
| 253 |
+
"aten.mm": 15134654464
|
| 254 |
+
}
|
| 255 |
+
}
|
| 256 |
+
],
|
| 257 |
+
"mean_prefill_gflops_counted": 10167.670253184,
|
| 258 |
+
"mean_decode_one_token_gflops_counted": 15.435464832,
|
| 259 |
+
"scope": "HF reference execution on the A800 with real task inputs; dense products/convolutions/supported attention counted with multiply-add=2 FLOPs. vLLM custom kernels, elementwise/normalization/nonlinearities and other unsupported operations are not complete.",
|
| 260 |
+
"answer_generation": "not performed; one known-token decode probe only"
|
| 261 |
+
}
|
releases/20260913_lr/paper_data/main_table_cost_20260912/components/minicpm_latency.json
ADDED
|
@@ -0,0 +1,2105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"model_kind": "MiniCPM",
|
| 4 |
+
"task": "grefcoco",
|
| 5 |
+
"expert": "instancevg",
|
| 6 |
+
"model_path": "/mnt/HithinkOmniSSD/user_workspace/caisihang/checkpoints/MiniCPM-V-4_5",
|
| 7 |
+
"device": "cuda:0",
|
| 8 |
+
"gpu_memory_utilization": 0.85,
|
| 9 |
+
"sample_count_requested": 66,
|
| 10 |
+
"warmup_count": 2,
|
| 11 |
+
"measured_count": 64,
|
| 12 |
+
"batch_size": 1,
|
| 13 |
+
"sample_ids": [
|
| 14 |
+
"testA-29199-80373",
|
| 15 |
+
"testB-44351-122075",
|
| 16 |
+
"testA-91850-253019",
|
| 17 |
+
"testB-36888-101641",
|
| 18 |
+
"testA-54586-150233",
|
| 19 |
+
"testB-79644-219272",
|
| 20 |
+
"testB-65935-181776",
|
| 21 |
+
"testA-81622-224664",
|
| 22 |
+
"testA-10078-27739",
|
| 23 |
+
"testA-15582-42797",
|
| 24 |
+
"testA-5038-13943",
|
| 25 |
+
"testA-30844-84861",
|
| 26 |
+
"testB-15419-42356",
|
| 27 |
+
"testB-27486-75580",
|
| 28 |
+
"testB-945-2632",
|
| 29 |
+
"testA-24438-67190",
|
| 30 |
+
"testB-66161-182405",
|
| 31 |
+
"testB-42726-117659",
|
| 32 |
+
"testB-69439-191450",
|
| 33 |
+
"testA-54307-149426",
|
| 34 |
+
"testB-37669-103831",
|
| 35 |
+
"testB-50844-140105",
|
| 36 |
+
"testA-79438-218690",
|
| 37 |
+
"testA-62940-173454",
|
| 38 |
+
"testA-81245-223632",
|
| 39 |
+
"testA-47070-129599",
|
| 40 |
+
"testB-12381-34098",
|
| 41 |
+
"testA-44142-121498",
|
| 42 |
+
"testA-73258-201893",
|
| 43 |
+
"testB-46644-128436",
|
| 44 |
+
"testA-43889-120792",
|
| 45 |
+
"testB-36082-99363",
|
| 46 |
+
"testB-33108-91152",
|
| 47 |
+
"testA-26856-73830",
|
| 48 |
+
"testA-81271-223709",
|
| 49 |
+
"testA-54554-150141",
|
| 50 |
+
"testB-38636-106481",
|
| 51 |
+
"testA-5734-15837",
|
| 52 |
+
"testB-67558-186325",
|
| 53 |
+
"testB-47053-129554",
|
| 54 |
+
"testA-93064-256422",
|
| 55 |
+
"testB-60557-166821",
|
| 56 |
+
"testA-55116-151688",
|
| 57 |
+
"testB-65796-181392",
|
| 58 |
+
"testB-23014-63278",
|
| 59 |
+
"testA-24562-67515",
|
| 60 |
+
"testA-17736-48684",
|
| 61 |
+
"testA-58475-160949",
|
| 62 |
+
"testA-80455-221493",
|
| 63 |
+
"testA-31763-87405",
|
| 64 |
+
"testB-23920-65742",
|
| 65 |
+
"testA-469-1309",
|
| 66 |
+
"testB-56691-155998",
|
| 67 |
+
"testB-31493-86668",
|
| 68 |
+
"testA-59284-163254",
|
| 69 |
+
"testA-57397-158015",
|
| 70 |
+
"testB-2935-8111",
|
| 71 |
+
"testB-22290-61226",
|
| 72 |
+
"testB-8416-23140",
|
| 73 |
+
"testA-60731-167320",
|
| 74 |
+
"testA-49055-135086",
|
| 75 |
+
"testA-7857-21644",
|
| 76 |
+
"testB-79645-219273",
|
| 77 |
+
"testB-52921-145707",
|
| 78 |
+
"testB-69479-191562",
|
| 79 |
+
"testA-69466-191524"
|
| 80 |
+
],
|
| 81 |
+
"batches": [
|
| 82 |
+
{
|
| 83 |
+
"sample_ids": [
|
| 84 |
+
"testA-91850-253019"
|
| 85 |
+
],
|
| 86 |
+
"batch_size": 1,
|
| 87 |
+
"generation_latency_seconds": 0.8861644599819556,
|
| 88 |
+
"batch_wall_latency_seconds": 0.9864439839730039,
|
| 89 |
+
"amortized_generation_ms_per_sample": 886.1644599819556,
|
| 90 |
+
"amortized_batch_wall_ms_per_sample": 986.4439839730039,
|
| 91 |
+
"outputs": [
|
| 92 |
+
{
|
| 93 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[420 ,220 ,710 ,640 ],\"score\":0.95}]}",
|
| 94 |
+
"valid_prediction": {
|
| 95 |
+
"objects": [
|
| 96 |
+
{
|
| 97 |
+
"bbox_2d": [
|
| 98 |
+
420,
|
| 99 |
+
220,
|
| 100 |
+
710,
|
| 101 |
+
640
|
| 102 |
+
],
|
| 103 |
+
"score": 0.95
|
| 104 |
+
}
|
| 105 |
+
]
|
| 106 |
+
},
|
| 107 |
+
"error": null,
|
| 108 |
+
"finish_reason": "stop",
|
| 109 |
+
"completion_tokens": 39,
|
| 110 |
+
"prompt_tokens": 530
|
| 111 |
+
}
|
| 112 |
+
]
|
| 113 |
+
},
|
| 114 |
+
{
|
| 115 |
+
"sample_ids": [
|
| 116 |
+
"testB-36888-101641"
|
| 117 |
+
],
|
| 118 |
+
"batch_size": 1,
|
| 119 |
+
"generation_latency_seconds": 1.3824354230891913,
|
| 120 |
+
"batch_wall_latency_seconds": 1.3937634559115395,
|
| 121 |
+
"amortized_generation_ms_per_sample": 1382.4354230891913,
|
| 122 |
+
"amortized_batch_wall_ms_per_sample": 1393.7634559115395,
|
| 123 |
+
"outputs": [
|
| 124 |
+
{
|
| 125 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[30,570,720,900],\"score\":0.95},{\"bbox_2d\":[390,260,960,860],\"score\":0.93}]}",
|
| 126 |
+
"valid_prediction": {
|
| 127 |
+
"objects": [
|
| 128 |
+
{
|
| 129 |
+
"bbox_2d": [
|
| 130 |
+
30,
|
| 131 |
+
570,
|
| 132 |
+
720,
|
| 133 |
+
900
|
| 134 |
+
],
|
| 135 |
+
"score": 0.95
|
| 136 |
+
},
|
| 137 |
+
{
|
| 138 |
+
"bbox_2d": [
|
| 139 |
+
390,
|
| 140 |
+
260,
|
| 141 |
+
960,
|
| 142 |
+
860
|
| 143 |
+
],
|
| 144 |
+
"score": 0.93
|
| 145 |
+
}
|
| 146 |
+
]
|
| 147 |
+
},
|
| 148 |
+
"error": null,
|
| 149 |
+
"finish_reason": "stop",
|
| 150 |
+
"completion_tokens": 60,
|
| 151 |
+
"prompt_tokens": 528
|
| 152 |
+
}
|
| 153 |
+
]
|
| 154 |
+
},
|
| 155 |
+
{
|
| 156 |
+
"sample_ids": [
|
| 157 |
+
"testA-54586-150233"
|
| 158 |
+
],
|
| 159 |
+
"batch_size": 1,
|
| 160 |
+
"generation_latency_seconds": 10.028413712978363,
|
| 161 |
+
"batch_wall_latency_seconds": 10.048457957920618,
|
| 162 |
+
"amortized_generation_ms_per_sample": 10028.413712978363,
|
| 163 |
+
"amortized_batch_wall_ms_per_sample": 10048.457957920618,
|
| 164 |
+
"outputs": [
|
| 165 |
+
{
|
| 166 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[480 ",
|
| 167 |
+
"valid_prediction": null,
|
| 168 |
+
"error": "Expecting ',' delimiter: line 1 column 529 (char 528)",
|
| 169 |
+
"finish_reason": "length",
|
| 170 |
+
"completion_tokens": 512,
|
| 171 |
+
"prompt_tokens": 660
|
| 172 |
+
}
|
| 173 |
+
]
|
| 174 |
+
},
|
| 175 |
+
{
|
| 176 |
+
"sample_ids": [
|
| 177 |
+
"testB-79644-219272"
|
| 178 |
+
],
|
| 179 |
+
"batch_size": 1,
|
| 180 |
+
"generation_latency_seconds": 7.480462849023752,
|
| 181 |
+
"batch_wall_latency_seconds": 7.510920200962573,
|
| 182 |
+
"amortized_generation_ms_per_sample": 7480.462849023752,
|
| 183 |
+
"amortized_batch_wall_ms_per_sample": 7510.920200962573,
|
| 184 |
+
"outputs": [
|
| 185 |
+
{
|
| 186 |
+
"raw_response": "{\"objects\": [{\"bbox_2d\":[120,390,270,790],\"score\":0.92},{\"bbox_2d\":[160,430,300,730],\"score\":0.91},{\"bbox_2d\":[270,560,320,740],\"score\":0.90},{\"bbox_2d\":[300,550,350,720],\"score\":0.89},{\"bbox_2d\":[340,540,380,700],\"score\":0.88},{\"bbox_2d\":[450,560,500,770],\"score\":0.87},{\"bbox_2d\":[480,580,530,760],\"score\":0.86},{\"bbox_2d\":[530,560,570,710],\"score\":0.85},{\"bbox_2d\":[570,530,740,780],\"score\":0.84},{\"bbox_2d\":[670,550,740,810],\"score\":0.83},{\"bbox_2d\":[740,540,800,760],\"score\":0.82},{\"bbox_2d\":[800,540,840,700],\"score\":0.81},{\"bbox_2d\":[840,540,900,780],\"score\":0.80}]}",
|
| 187 |
+
"valid_prediction": {
|
| 188 |
+
"objects": [
|
| 189 |
+
{
|
| 190 |
+
"bbox_2d": [
|
| 191 |
+
120,
|
| 192 |
+
390,
|
| 193 |
+
270,
|
| 194 |
+
790
|
| 195 |
+
],
|
| 196 |
+
"score": 0.92
|
| 197 |
+
},
|
| 198 |
+
{
|
| 199 |
+
"bbox_2d": [
|
| 200 |
+
160,
|
| 201 |
+
430,
|
| 202 |
+
300,
|
| 203 |
+
730
|
| 204 |
+
],
|
| 205 |
+
"score": 0.91
|
| 206 |
+
},
|
| 207 |
+
{
|
| 208 |
+
"bbox_2d": [
|
| 209 |
+
270,
|
| 210 |
+
560,
|
| 211 |
+
320,
|
| 212 |
+
740
|
| 213 |
+
],
|
| 214 |
+
"score": 0.9
|
| 215 |
+
},
|
| 216 |
+
{
|
| 217 |
+
"bbox_2d": [
|
| 218 |
+
300,
|
| 219 |
+
550,
|
| 220 |
+
350,
|
| 221 |
+
720
|
| 222 |
+
],
|
| 223 |
+
"score": 0.89
|
| 224 |
+
},
|
| 225 |
+
{
|
| 226 |
+
"bbox_2d": [
|
| 227 |
+
340,
|
| 228 |
+
540,
|
| 229 |
+
380,
|
| 230 |
+
700
|
| 231 |
+
],
|
| 232 |
+
"score": 0.88
|
| 233 |
+
},
|
| 234 |
+
{
|
| 235 |
+
"bbox_2d": [
|
| 236 |
+
450,
|
| 237 |
+
560,
|
| 238 |
+
500,
|
| 239 |
+
770
|
| 240 |
+
],
|
| 241 |
+
"score": 0.87
|
| 242 |
+
},
|
| 243 |
+
{
|
| 244 |
+
"bbox_2d": [
|
| 245 |
+
480,
|
| 246 |
+
580,
|
| 247 |
+
530,
|
| 248 |
+
760
|
| 249 |
+
],
|
| 250 |
+
"score": 0.86
|
| 251 |
+
},
|
| 252 |
+
{
|
| 253 |
+
"bbox_2d": [
|
| 254 |
+
530,
|
| 255 |
+
560,
|
| 256 |
+
570,
|
| 257 |
+
710
|
| 258 |
+
],
|
| 259 |
+
"score": 0.85
|
| 260 |
+
},
|
| 261 |
+
{
|
| 262 |
+
"bbox_2d": [
|
| 263 |
+
570,
|
| 264 |
+
530,
|
| 265 |
+
740,
|
| 266 |
+
780
|
| 267 |
+
],
|
| 268 |
+
"score": 0.84
|
| 269 |
+
},
|
| 270 |
+
{
|
| 271 |
+
"bbox_2d": [
|
| 272 |
+
670,
|
| 273 |
+
550,
|
| 274 |
+
740,
|
| 275 |
+
810
|
| 276 |
+
],
|
| 277 |
+
"score": 0.83
|
| 278 |
+
},
|
| 279 |
+
{
|
| 280 |
+
"bbox_2d": [
|
| 281 |
+
740,
|
| 282 |
+
540,
|
| 283 |
+
800,
|
| 284 |
+
760
|
| 285 |
+
],
|
| 286 |
+
"score": 0.82
|
| 287 |
+
},
|
| 288 |
+
{
|
| 289 |
+
"bbox_2d": [
|
| 290 |
+
800,
|
| 291 |
+
540,
|
| 292 |
+
840,
|
| 293 |
+
700
|
| 294 |
+
],
|
| 295 |
+
"score": 0.81
|
| 296 |
+
},
|
| 297 |
+
{
|
| 298 |
+
"bbox_2d": [
|
| 299 |
+
840,
|
| 300 |
+
540,
|
| 301 |
+
900,
|
| 302 |
+
780
|
| 303 |
+
],
|
| 304 |
+
"score": 0.8
|
| 305 |
+
}
|
| 306 |
+
]
|
| 307 |
+
},
|
| 308 |
+
"error": null,
|
| 309 |
+
"finish_reason": "stop",
|
| 310 |
+
"completion_tokens": 370,
|
| 311 |
+
"prompt_tokens": 520
|
| 312 |
+
}
|
| 313 |
+
]
|
| 314 |
+
},
|
| 315 |
+
{
|
| 316 |
+
"sample_ids": [
|
| 317 |
+
"testB-65935-181776"
|
| 318 |
+
],
|
| 319 |
+
"batch_size": 1,
|
| 320 |
+
"generation_latency_seconds": 10.180860323016532,
|
| 321 |
+
"batch_wall_latency_seconds": 10.195898448000662,
|
| 322 |
+
"amortized_generation_ms_per_sample": 10180.860323016532,
|
| 323 |
+
"amortized_batch_wall_ms_per_sample": 10195.898448000662,
|
| 324 |
+
"outputs": [
|
| 325 |
+
{
|
| 326 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[340 ",
|
| 327 |
+
"valid_prediction": null,
|
| 328 |
+
"error": "Expecting ',' delimiter: line 1 column 529 (char 528)",
|
| 329 |
+
"finish_reason": "length",
|
| 330 |
+
"completion_tokens": 512,
|
| 331 |
+
"prompt_tokens": 524
|
| 332 |
+
}
|
| 333 |
+
]
|
| 334 |
+
},
|
| 335 |
+
{
|
| 336 |
+
"sample_ids": [
|
| 337 |
+
"testA-81622-224664"
|
| 338 |
+
],
|
| 339 |
+
"batch_size": 1,
|
| 340 |
+
"generation_latency_seconds": 0.2510422539198771,
|
| 341 |
+
"batch_wall_latency_seconds": 0.26205689006019384,
|
| 342 |
+
"amortized_generation_ms_per_sample": 251.0422539198771,
|
| 343 |
+
"amortized_batch_wall_ms_per_sample": 262.05689006019384,
|
| 344 |
+
"outputs": [
|
| 345 |
+
{
|
| 346 |
+
"raw_response": "{\"objects\":[]}",
|
| 347 |
+
"valid_prediction": {
|
| 348 |
+
"objects": []
|
| 349 |
+
},
|
| 350 |
+
"error": null,
|
| 351 |
+
"finish_reason": "stop",
|
| 352 |
+
"completion_tokens": 6,
|
| 353 |
+
"prompt_tokens": 524
|
| 354 |
+
}
|
| 355 |
+
]
|
| 356 |
+
},
|
| 357 |
+
{
|
| 358 |
+
"sample_ids": [
|
| 359 |
+
"testA-10078-27739"
|
| 360 |
+
],
|
| 361 |
+
"batch_size": 1,
|
| 362 |
+
"generation_latency_seconds": 0.26944356702733785,
|
| 363 |
+
"batch_wall_latency_seconds": 0.2834722528932616,
|
| 364 |
+
"amortized_generation_ms_per_sample": 269.44356702733785,
|
| 365 |
+
"amortized_batch_wall_ms_per_sample": 283.4722528932616,
|
| 366 |
+
"outputs": [
|
| 367 |
+
{
|
| 368 |
+
"raw_response": "{\"objects\":[]}",
|
| 369 |
+
"valid_prediction": {
|
| 370 |
+
"objects": []
|
| 371 |
+
},
|
| 372 |
+
"error": null,
|
| 373 |
+
"finish_reason": "stop",
|
| 374 |
+
"completion_tokens": 6,
|
| 375 |
+
"prompt_tokens": 525
|
| 376 |
+
}
|
| 377 |
+
]
|
| 378 |
+
},
|
| 379 |
+
{
|
| 380 |
+
"sample_ids": [
|
| 381 |
+
"testA-15582-42797"
|
| 382 |
+
],
|
| 383 |
+
"batch_size": 1,
|
| 384 |
+
"generation_latency_seconds": 1.9161002039909363,
|
| 385 |
+
"batch_wall_latency_seconds": 1.9308117219479755,
|
| 386 |
+
"amortized_generation_ms_per_sample": 1916.1002039909363,
|
| 387 |
+
"amortized_batch_wall_ms_per_sample": 1930.8117219479755,
|
| 388 |
+
"outputs": [
|
| 389 |
+
{
|
| 390 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[80,280,400,980],\"score\":0.95},{\"bbox_2d\":[310,390,510,980],\"score\":0.93},{\"bbox_2d\":[540,400,640,870],\"score\":0.88}]}",
|
| 391 |
+
"valid_prediction": {
|
| 392 |
+
"objects": [
|
| 393 |
+
{
|
| 394 |
+
"bbox_2d": [
|
| 395 |
+
80,
|
| 396 |
+
280,
|
| 397 |
+
400,
|
| 398 |
+
980
|
| 399 |
+
],
|
| 400 |
+
"score": 0.95
|
| 401 |
+
},
|
| 402 |
+
{
|
| 403 |
+
"bbox_2d": [
|
| 404 |
+
310,
|
| 405 |
+
390,
|
| 406 |
+
510,
|
| 407 |
+
980
|
| 408 |
+
],
|
| 409 |
+
"score": 0.93
|
| 410 |
+
},
|
| 411 |
+
{
|
| 412 |
+
"bbox_2d": [
|
| 413 |
+
540,
|
| 414 |
+
400,
|
| 415 |
+
640,
|
| 416 |
+
870
|
| 417 |
+
],
|
| 418 |
+
"score": 0.88
|
| 419 |
+
}
|
| 420 |
+
]
|
| 421 |
+
},
|
| 422 |
+
"error": null,
|
| 423 |
+
"finish_reason": "stop",
|
| 424 |
+
"completion_tokens": 88,
|
| 425 |
+
"prompt_tokens": 539
|
| 426 |
+
}
|
| 427 |
+
]
|
| 428 |
+
},
|
| 429 |
+
{
|
| 430 |
+
"sample_ids": [
|
| 431 |
+
"testA-5038-13943"
|
| 432 |
+
],
|
| 433 |
+
"batch_size": 1,
|
| 434 |
+
"generation_latency_seconds": 0.7828407019842416,
|
| 435 |
+
"batch_wall_latency_seconds": 0.7960362600861117,
|
| 436 |
+
"amortized_generation_ms_per_sample": 782.8407019842416,
|
| 437 |
+
"amortized_batch_wall_ms_per_sample": 796.0362600861117,
|
| 438 |
+
"outputs": [
|
| 439 |
+
{
|
| 440 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[490,240,780,980],\"score\":0.95}]}",
|
| 441 |
+
"valid_prediction": {
|
| 442 |
+
"objects": [
|
| 443 |
+
{
|
| 444 |
+
"bbox_2d": [
|
| 445 |
+
490,
|
| 446 |
+
240,
|
| 447 |
+
780,
|
| 448 |
+
980
|
| 449 |
+
],
|
| 450 |
+
"score": 0.95
|
| 451 |
+
}
|
| 452 |
+
]
|
| 453 |
+
},
|
| 454 |
+
"error": null,
|
| 455 |
+
"finish_reason": "stop",
|
| 456 |
+
"completion_tokens": 33,
|
| 457 |
+
"prompt_tokens": 524
|
| 458 |
+
}
|
| 459 |
+
]
|
| 460 |
+
},
|
| 461 |
+
{
|
| 462 |
+
"sample_ids": [
|
| 463 |
+
"testA-30844-84861"
|
| 464 |
+
],
|
| 465 |
+
"batch_size": 1,
|
| 466 |
+
"generation_latency_seconds": 10.480434139957651,
|
| 467 |
+
"batch_wall_latency_seconds": 10.485813157982193,
|
| 468 |
+
"amortized_generation_ms_per_sample": 10480.434139957651,
|
| 469 |
+
"amortized_batch_wall_ms_per_sample": 10485.813157982193,
|
| 470 |
+
"outputs": [
|
| 471 |
+
{
|
| 472 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[470 \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n",
|
| 473 |
+
"valid_prediction": null,
|
| 474 |
+
"error": "Expecting ',' delimiter: line 499 column 1 (char 2020)",
|
| 475 |
+
"finish_reason": "length",
|
| 476 |
+
"completion_tokens": 512,
|
| 477 |
+
"prompt_tokens": 543
|
| 478 |
+
}
|
| 479 |
+
]
|
| 480 |
+
},
|
| 481 |
+
{
|
| 482 |
+
"sample_ids": [
|
| 483 |
+
"testB-15419-42356"
|
| 484 |
+
],
|
| 485 |
+
"batch_size": 1,
|
| 486 |
+
"generation_latency_seconds": 10.714943284052424,
|
| 487 |
+
"batch_wall_latency_seconds": 10.730338808032684,
|
| 488 |
+
"amortized_generation_ms_per_sample": 10714.943284052424,
|
| 489 |
+
"amortized_batch_wall_ms_per_sample": 10730.338808032684,
|
| 490 |
+
"outputs": [
|
| 491 |
+
{
|
| 492 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[110 ",
|
| 493 |
+
"valid_prediction": null,
|
| 494 |
+
"error": "Expecting ',' delimiter: line 1 column 529 (char 528)",
|
| 495 |
+
"finish_reason": "length",
|
| 496 |
+
"completion_tokens": 512,
|
| 497 |
+
"prompt_tokens": 526
|
| 498 |
+
}
|
| 499 |
+
]
|
| 500 |
+
},
|
| 501 |
+
{
|
| 502 |
+
"sample_ids": [
|
| 503 |
+
"testB-27486-75580"
|
| 504 |
+
],
|
| 505 |
+
"batch_size": 1,
|
| 506 |
+
"generation_latency_seconds": 10.180824494105764,
|
| 507 |
+
"batch_wall_latency_seconds": 10.188868912984617,
|
| 508 |
+
"amortized_generation_ms_per_sample": 10180.824494105764,
|
| 509 |
+
"amortized_batch_wall_ms_per_sample": 10188.868912984617,
|
| 510 |
+
"outputs": [
|
| 511 |
+
{
|
| 512 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[530 \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n",
|
| 513 |
+
"valid_prediction": null,
|
| 514 |
+
"error": "Expecting ',' delimiter: line 499 column 1 (char 2020)",
|
| 515 |
+
"finish_reason": "length",
|
| 516 |
+
"completion_tokens": 512,
|
| 517 |
+
"prompt_tokens": 530
|
| 518 |
+
}
|
| 519 |
+
]
|
| 520 |
+
},
|
| 521 |
+
{
|
| 522 |
+
"sample_ids": [
|
| 523 |
+
"testB-945-2632"
|
| 524 |
+
],
|
| 525 |
+
"batch_size": 1,
|
| 526 |
+
"generation_latency_seconds": 3.843070151982829,
|
| 527 |
+
"batch_wall_latency_seconds": 3.859969821991399,
|
| 528 |
+
"amortized_generation_ms_per_sample": 3843.070151982829,
|
| 529 |
+
"amortized_batch_wall_ms_per_sample": 3859.969821991399,
|
| 530 |
+
"outputs": [
|
| 531 |
+
{
|
| 532 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[120,530,220,910] ,\"score\":0.92},{\"bbox_2d\":[250,530,350,910] ,\"score\":0.92},{\"bbox_2d\":[380,530,480,910] ,\"score\":0.92},{\"bbox_2d\":[500,520,600,910] ,\"score\":0.92},{\"bbox_2d\":[620,530,720,910] ,\"score\":0.92},{\"bbox_2d\":[740,530,840,910] ,\"score\":0.92}]}",
|
| 533 |
+
"valid_prediction": {
|
| 534 |
+
"objects": [
|
| 535 |
+
{
|
| 536 |
+
"bbox_2d": [
|
| 537 |
+
120,
|
| 538 |
+
530,
|
| 539 |
+
220,
|
| 540 |
+
910
|
| 541 |
+
],
|
| 542 |
+
"score": 0.92
|
| 543 |
+
},
|
| 544 |
+
{
|
| 545 |
+
"bbox_2d": [
|
| 546 |
+
250,
|
| 547 |
+
530,
|
| 548 |
+
350,
|
| 549 |
+
910
|
| 550 |
+
],
|
| 551 |
+
"score": 0.92
|
| 552 |
+
},
|
| 553 |
+
{
|
| 554 |
+
"bbox_2d": [
|
| 555 |
+
380,
|
| 556 |
+
530,
|
| 557 |
+
480,
|
| 558 |
+
910
|
| 559 |
+
],
|
| 560 |
+
"score": 0.92
|
| 561 |
+
},
|
| 562 |
+
{
|
| 563 |
+
"bbox_2d": [
|
| 564 |
+
500,
|
| 565 |
+
520,
|
| 566 |
+
600,
|
| 567 |
+
910
|
| 568 |
+
],
|
| 569 |
+
"score": 0.92
|
| 570 |
+
},
|
| 571 |
+
{
|
| 572 |
+
"bbox_2d": [
|
| 573 |
+
620,
|
| 574 |
+
530,
|
| 575 |
+
720,
|
| 576 |
+
910
|
| 577 |
+
],
|
| 578 |
+
"score": 0.92
|
| 579 |
+
},
|
| 580 |
+
{
|
| 581 |
+
"bbox_2d": [
|
| 582 |
+
740,
|
| 583 |
+
530,
|
| 584 |
+
840,
|
| 585 |
+
910
|
| 586 |
+
],
|
| 587 |
+
"score": 0.92
|
| 588 |
+
}
|
| 589 |
+
]
|
| 590 |
+
},
|
| 591 |
+
"error": null,
|
| 592 |
+
"finish_reason": "stop",
|
| 593 |
+
"completion_tokens": 179,
|
| 594 |
+
"prompt_tokens": 530
|
| 595 |
+
}
|
| 596 |
+
]
|
| 597 |
+
},
|
| 598 |
+
{
|
| 599 |
+
"sample_ids": [
|
| 600 |
+
"testA-24438-67190"
|
| 601 |
+
],
|
| 602 |
+
"batch_size": 1,
|
| 603 |
+
"generation_latency_seconds": 10.143939319998026,
|
| 604 |
+
"batch_wall_latency_seconds": 10.162826716084965,
|
| 605 |
+
"amortized_generation_ms_per_sample": 10143.939319998026,
|
| 606 |
+
"amortized_batch_wall_ms_per_sample": 10162.826716084965,
|
| 607 |
+
"outputs": [
|
| 608 |
+
{
|
| 609 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[290,240,590,920],\"score\":0.95},{\"bbox_2d\":[730 \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n",
|
| 610 |
+
"valid_prediction": null,
|
| 611 |
+
"error": "Expecting ',' delimiter: line 472 column 1 (char 1954)",
|
| 612 |
+
"finish_reason": "length",
|
| 613 |
+
"completion_tokens": 512,
|
| 614 |
+
"prompt_tokens": 525
|
| 615 |
+
}
|
| 616 |
+
]
|
| 617 |
+
},
|
| 618 |
+
{
|
| 619 |
+
"sample_ids": [
|
| 620 |
+
"testB-66161-182405"
|
| 621 |
+
],
|
| 622 |
+
"batch_size": 1,
|
| 623 |
+
"generation_latency_seconds": 10.25821736396756,
|
| 624 |
+
"batch_wall_latency_seconds": 10.271643411950208,
|
| 625 |
+
"amortized_generation_ms_per_sample": 10258.21736396756,
|
| 626 |
+
"amortized_batch_wall_ms_per_sample": 10271.643411950208,
|
| 627 |
+
"outputs": [
|
| 628 |
+
{
|
| 629 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[100 ",
|
| 630 |
+
"valid_prediction": null,
|
| 631 |
+
"error": "Expecting ',' delimiter: line 1 column 529 (char 528)",
|
| 632 |
+
"finish_reason": "length",
|
| 633 |
+
"completion_tokens": 512,
|
| 634 |
+
"prompt_tokens": 523
|
| 635 |
+
}
|
| 636 |
+
]
|
| 637 |
+
},
|
| 638 |
+
{
|
| 639 |
+
"sample_ids": [
|
| 640 |
+
"testB-42726-117659"
|
| 641 |
+
],
|
| 642 |
+
"batch_size": 1,
|
| 643 |
+
"generation_latency_seconds": 0.9900629689218476,
|
| 644 |
+
"batch_wall_latency_seconds": 1.0045652500120923,
|
| 645 |
+
"amortized_generation_ms_per_sample": 990.0629689218476,
|
| 646 |
+
"amortized_batch_wall_ms_per_sample": 1004.5652500120923,
|
| 647 |
+
"outputs": [
|
| 648 |
+
{
|
| 649 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[775 ,775 ,999 ,999 ],\"score\":0.92}]}",
|
| 650 |
+
"valid_prediction": {
|
| 651 |
+
"objects": [
|
| 652 |
+
{
|
| 653 |
+
"bbox_2d": [
|
| 654 |
+
775,
|
| 655 |
+
775,
|
| 656 |
+
999,
|
| 657 |
+
999
|
| 658 |
+
],
|
| 659 |
+
"score": 0.92
|
| 660 |
+
}
|
| 661 |
+
]
|
| 662 |
+
},
|
| 663 |
+
"error": null,
|
| 664 |
+
"finish_reason": "stop",
|
| 665 |
+
"completion_tokens": 43,
|
| 666 |
+
"prompt_tokens": 525
|
| 667 |
+
}
|
| 668 |
+
]
|
| 669 |
+
},
|
| 670 |
+
{
|
| 671 |
+
"sample_ids": [
|
| 672 |
+
"testB-69439-191450"
|
| 673 |
+
],
|
| 674 |
+
"batch_size": 1,
|
| 675 |
+
"generation_latency_seconds": 0.19075044698547572,
|
| 676 |
+
"batch_wall_latency_seconds": 0.20009391102939844,
|
| 677 |
+
"amortized_generation_ms_per_sample": 190.75044698547572,
|
| 678 |
+
"amortized_batch_wall_ms_per_sample": 200.09391102939844,
|
| 679 |
+
"outputs": [
|
| 680 |
+
{
|
| 681 |
+
"raw_response": "{\"objects\":[]}",
|
| 682 |
+
"valid_prediction": {
|
| 683 |
+
"objects": []
|
| 684 |
+
},
|
| 685 |
+
"error": null,
|
| 686 |
+
"finish_reason": "stop",
|
| 687 |
+
"completion_tokens": 5,
|
| 688 |
+
"prompt_tokens": 388
|
| 689 |
+
}
|
| 690 |
+
]
|
| 691 |
+
},
|
| 692 |
+
{
|
| 693 |
+
"sample_ids": [
|
| 694 |
+
"testA-54307-149426"
|
| 695 |
+
],
|
| 696 |
+
"batch_size": 1,
|
| 697 |
+
"generation_latency_seconds": 1.2734235550742596,
|
| 698 |
+
"batch_wall_latency_seconds": 1.2819705069996417,
|
| 699 |
+
"amortized_generation_ms_per_sample": 1273.4235550742596,
|
| 700 |
+
"amortized_batch_wall_ms_per_sample": 1281.9705069996417,
|
| 701 |
+
"outputs": [
|
| 702 |
+
{
|
| 703 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[170,0,560,420] ,\"score\":0.95},{\"bbox_2d\":[540,40,830,910] ,\"score\":0.96}]}",
|
| 704 |
+
"valid_prediction": {
|
| 705 |
+
"objects": [
|
| 706 |
+
{
|
| 707 |
+
"bbox_2d": [
|
| 708 |
+
170,
|
| 709 |
+
0,
|
| 710 |
+
560,
|
| 711 |
+
420
|
| 712 |
+
],
|
| 713 |
+
"score": 0.95
|
| 714 |
+
},
|
| 715 |
+
{
|
| 716 |
+
"bbox_2d": [
|
| 717 |
+
540,
|
| 718 |
+
40,
|
| 719 |
+
830,
|
| 720 |
+
910
|
| 721 |
+
],
|
| 722 |
+
"score": 0.96
|
| 723 |
+
}
|
| 724 |
+
]
|
| 725 |
+
},
|
| 726 |
+
"error": null,
|
| 727 |
+
"finish_reason": "stop",
|
| 728 |
+
"completion_tokens": 60,
|
| 729 |
+
"prompt_tokens": 524
|
| 730 |
+
}
|
| 731 |
+
]
|
| 732 |
+
},
|
| 733 |
+
{
|
| 734 |
+
"sample_ids": [
|
| 735 |
+
"testB-37669-103831"
|
| 736 |
+
],
|
| 737 |
+
"batch_size": 1,
|
| 738 |
+
"generation_latency_seconds": 0.21837474894709885,
|
| 739 |
+
"batch_wall_latency_seconds": 0.23633889202028513,
|
| 740 |
+
"amortized_generation_ms_per_sample": 218.37474894709885,
|
| 741 |
+
"amortized_batch_wall_ms_per_sample": 236.33889202028513,
|
| 742 |
+
"outputs": [
|
| 743 |
+
{
|
| 744 |
+
"raw_response": "{\"objects\":[]}",
|
| 745 |
+
"valid_prediction": {
|
| 746 |
+
"objects": []
|
| 747 |
+
},
|
| 748 |
+
"error": null,
|
| 749 |
+
"finish_reason": "stop",
|
| 750 |
+
"completion_tokens": 5,
|
| 751 |
+
"prompt_tokens": 528
|
| 752 |
+
}
|
| 753 |
+
]
|
| 754 |
+
},
|
| 755 |
+
{
|
| 756 |
+
"sample_ids": [
|
| 757 |
+
"testB-50844-140105"
|
| 758 |
+
],
|
| 759 |
+
"batch_size": 1,
|
| 760 |
+
"generation_latency_seconds": 0.23954864998813719,
|
| 761 |
+
"batch_wall_latency_seconds": 0.2525435209972784,
|
| 762 |
+
"amortized_generation_ms_per_sample": 239.54864998813719,
|
| 763 |
+
"amortized_batch_wall_ms_per_sample": 252.5435209972784,
|
| 764 |
+
"outputs": [
|
| 765 |
+
{
|
| 766 |
+
"raw_response": "{\"objects\":[]}",
|
| 767 |
+
"valid_prediction": {
|
| 768 |
+
"objects": []
|
| 769 |
+
},
|
| 770 |
+
"error": null,
|
| 771 |
+
"finish_reason": "stop",
|
| 772 |
+
"completion_tokens": 6,
|
| 773 |
+
"prompt_tokens": 522
|
| 774 |
+
}
|
| 775 |
+
]
|
| 776 |
+
},
|
| 777 |
+
{
|
| 778 |
+
"sample_ids": [
|
| 779 |
+
"testA-79438-218690"
|
| 780 |
+
],
|
| 781 |
+
"batch_size": 1,
|
| 782 |
+
"generation_latency_seconds": 0.7543689489830285,
|
| 783 |
+
"batch_wall_latency_seconds": 0.7701102939900011,
|
| 784 |
+
"amortized_generation_ms_per_sample": 754.3689489830285,
|
| 785 |
+
"amortized_batch_wall_ms_per_sample": 770.1102939900011,
|
| 786 |
+
"outputs": [
|
| 787 |
+
{
|
| 788 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[260,170,520,910],\"score\":0.95}]}",
|
| 789 |
+
"valid_prediction": {
|
| 790 |
+
"objects": [
|
| 791 |
+
{
|
| 792 |
+
"bbox_2d": [
|
| 793 |
+
260,
|
| 794 |
+
170,
|
| 795 |
+
520,
|
| 796 |
+
910
|
| 797 |
+
],
|
| 798 |
+
"score": 0.95
|
| 799 |
+
}
|
| 800 |
+
]
|
| 801 |
+
},
|
| 802 |
+
"error": null,
|
| 803 |
+
"finish_reason": "stop",
|
| 804 |
+
"completion_tokens": 33,
|
| 805 |
+
"prompt_tokens": 522
|
| 806 |
+
}
|
| 807 |
+
]
|
| 808 |
+
},
|
| 809 |
+
{
|
| 810 |
+
"sample_ids": [
|
| 811 |
+
"testA-62940-173454"
|
| 812 |
+
],
|
| 813 |
+
"batch_size": 1,
|
| 814 |
+
"generation_latency_seconds": 0.8060770370066166,
|
| 815 |
+
"batch_wall_latency_seconds": 0.8215904459357262,
|
| 816 |
+
"amortized_generation_ms_per_sample": 806.0770370066166,
|
| 817 |
+
"amortized_batch_wall_ms_per_sample": 821.5904459357262,
|
| 818 |
+
"outputs": [
|
| 819 |
+
{
|
| 820 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[620,270,750,690],\"score\":0.95}]}",
|
| 821 |
+
"valid_prediction": {
|
| 822 |
+
"objects": [
|
| 823 |
+
{
|
| 824 |
+
"bbox_2d": [
|
| 825 |
+
620,
|
| 826 |
+
270,
|
| 827 |
+
750,
|
| 828 |
+
690
|
| 829 |
+
],
|
| 830 |
+
"score": 0.95
|
| 831 |
+
}
|
| 832 |
+
]
|
| 833 |
+
},
|
| 834 |
+
"error": null,
|
| 835 |
+
"finish_reason": "stop",
|
| 836 |
+
"completion_tokens": 33,
|
| 837 |
+
"prompt_tokens": 525
|
| 838 |
+
}
|
| 839 |
+
]
|
| 840 |
+
},
|
| 841 |
+
{
|
| 842 |
+
"sample_ids": [
|
| 843 |
+
"testA-81245-223632"
|
| 844 |
+
],
|
| 845 |
+
"batch_size": 1,
|
| 846 |
+
"generation_latency_seconds": 2.46771560003981,
|
| 847 |
+
"batch_wall_latency_seconds": 2.4794425220461562,
|
| 848 |
+
"amortized_generation_ms_per_sample": 2467.71560003981,
|
| 849 |
+
"amortized_batch_wall_ms_per_sample": 2479.4425220461562,
|
| 850 |
+
"outputs": [
|
| 851 |
+
{
|
| 852 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[530,110,760,880] ,\"score\":0.95},{\"bbox_2d\":[560,180,630,610] ,\"score\":0.93},{\"bbox_2d\":[690,130,790,740] ,\"score\":0.91},{\"bbox_2d\":[790,110,920,660] ,\"score\":0.90}]}",
|
| 853 |
+
"valid_prediction": {
|
| 854 |
+
"objects": [
|
| 855 |
+
{
|
| 856 |
+
"bbox_2d": [
|
| 857 |
+
530,
|
| 858 |
+
110,
|
| 859 |
+
760,
|
| 860 |
+
880
|
| 861 |
+
],
|
| 862 |
+
"score": 0.95
|
| 863 |
+
},
|
| 864 |
+
{
|
| 865 |
+
"bbox_2d": [
|
| 866 |
+
560,
|
| 867 |
+
180,
|
| 868 |
+
630,
|
| 869 |
+
610
|
| 870 |
+
],
|
| 871 |
+
"score": 0.93
|
| 872 |
+
},
|
| 873 |
+
{
|
| 874 |
+
"bbox_2d": [
|
| 875 |
+
690,
|
| 876 |
+
130,
|
| 877 |
+
790,
|
| 878 |
+
740
|
| 879 |
+
],
|
| 880 |
+
"score": 0.91
|
| 881 |
+
},
|
| 882 |
+
{
|
| 883 |
+
"bbox_2d": [
|
| 884 |
+
790,
|
| 885 |
+
110,
|
| 886 |
+
920,
|
| 887 |
+
660
|
| 888 |
+
],
|
| 889 |
+
"score": 0.9
|
| 890 |
+
}
|
| 891 |
+
]
|
| 892 |
+
},
|
| 893 |
+
"error": null,
|
| 894 |
+
"finish_reason": "stop",
|
| 895 |
+
"completion_tokens": 121,
|
| 896 |
+
"prompt_tokens": 530
|
| 897 |
+
}
|
| 898 |
+
]
|
| 899 |
+
},
|
| 900 |
+
{
|
| 901 |
+
"sample_ids": [
|
| 902 |
+
"testA-47070-129599"
|
| 903 |
+
],
|
| 904 |
+
"batch_size": 1,
|
| 905 |
+
"generation_latency_seconds": 0.22282294591423124,
|
| 906 |
+
"batch_wall_latency_seconds": 0.2299416270107031,
|
| 907 |
+
"amortized_generation_ms_per_sample": 222.82294591423124,
|
| 908 |
+
"amortized_batch_wall_ms_per_sample": 229.9416270107031,
|
| 909 |
+
"outputs": [
|
| 910 |
+
{
|
| 911 |
+
"raw_response": "{\"objects\":[]}",
|
| 912 |
+
"valid_prediction": {
|
| 913 |
+
"objects": []
|
| 914 |
+
},
|
| 915 |
+
"error": null,
|
| 916 |
+
"finish_reason": "stop",
|
| 917 |
+
"completion_tokens": 5,
|
| 918 |
+
"prompt_tokens": 521
|
| 919 |
+
}
|
| 920 |
+
]
|
| 921 |
+
},
|
| 922 |
+
{
|
| 923 |
+
"sample_ids": [
|
| 924 |
+
"testB-12381-34098"
|
| 925 |
+
],
|
| 926 |
+
"batch_size": 1,
|
| 927 |
+
"generation_latency_seconds": 10.451599241001531,
|
| 928 |
+
"batch_wall_latency_seconds": 10.457325526047498,
|
| 929 |
+
"amortized_generation_ms_per_sample": 10451.599241001531,
|
| 930 |
+
"amortized_batch_wall_ms_per_sample": 10457.325526047498,
|
| 931 |
+
"outputs": [
|
| 932 |
+
{
|
| 933 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[290 ",
|
| 934 |
+
"valid_prediction": null,
|
| 935 |
+
"error": "Expecting ',' delimiter: line 1 column 529 (char 528)",
|
| 936 |
+
"finish_reason": "length",
|
| 937 |
+
"completion_tokens": 512,
|
| 938 |
+
"prompt_tokens": 534
|
| 939 |
+
}
|
| 940 |
+
]
|
| 941 |
+
},
|
| 942 |
+
{
|
| 943 |
+
"sample_ids": [
|
| 944 |
+
"testA-44142-121498"
|
| 945 |
+
],
|
| 946 |
+
"batch_size": 1,
|
| 947 |
+
"generation_latency_seconds": 1.4406404530163854,
|
| 948 |
+
"batch_wall_latency_seconds": 1.4515395000344142,
|
| 949 |
+
"amortized_generation_ms_per_sample": 1440.6404530163854,
|
| 950 |
+
"amortized_batch_wall_ms_per_sample": 1451.5395000344142,
|
| 951 |
+
"outputs": [
|
| 952 |
+
{
|
| 953 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[0,100,580,980] ,\"score\":0.95},{\"bbox_2d\":[570,120,1000,900] ,\"score\":0.93}]}",
|
| 954 |
+
"valid_prediction": {
|
| 955 |
+
"objects": [
|
| 956 |
+
{
|
| 957 |
+
"bbox_2d": [
|
| 958 |
+
0,
|
| 959 |
+
100,
|
| 960 |
+
580,
|
| 961 |
+
980
|
| 962 |
+
],
|
| 963 |
+
"score": 0.95
|
| 964 |
+
},
|
| 965 |
+
{
|
| 966 |
+
"bbox_2d": [
|
| 967 |
+
570,
|
| 968 |
+
120,
|
| 969 |
+
1000,
|
| 970 |
+
900
|
| 971 |
+
],
|
| 972 |
+
"score": 0.93
|
| 973 |
+
}
|
| 974 |
+
]
|
| 975 |
+
},
|
| 976 |
+
"error": null,
|
| 977 |
+
"finish_reason": "stop",
|
| 978 |
+
"completion_tokens": 62,
|
| 979 |
+
"prompt_tokens": 523
|
| 980 |
+
}
|
| 981 |
+
]
|
| 982 |
+
},
|
| 983 |
+
{
|
| 984 |
+
"sample_ids": [
|
| 985 |
+
"testA-73258-201893"
|
| 986 |
+
],
|
| 987 |
+
"batch_size": 1,
|
| 988 |
+
"generation_latency_seconds": 11.113033535890281,
|
| 989 |
+
"batch_wall_latency_seconds": 11.123388323001564,
|
| 990 |
+
"amortized_generation_ms_per_sample": 11113.033535890281,
|
| 991 |
+
"amortized_batch_wall_ms_per_sample": 11123.388323001564,
|
| 992 |
+
"outputs": [
|
| 993 |
+
{
|
| 994 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[520 \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n",
|
| 995 |
+
"valid_prediction": null,
|
| 996 |
+
"error": "Expecting ',' delimiter: line 251 column 1 (char 1401)",
|
| 997 |
+
"finish_reason": "length",
|
| 998 |
+
"completion_tokens": 512,
|
| 999 |
+
"prompt_tokens": 526
|
| 1000 |
+
}
|
| 1001 |
+
]
|
| 1002 |
+
},
|
| 1003 |
+
{
|
| 1004 |
+
"sample_ids": [
|
| 1005 |
+
"testB-46644-128436"
|
| 1006 |
+
],
|
| 1007 |
+
"batch_size": 1,
|
| 1008 |
+
"generation_latency_seconds": 1.2878015459282324,
|
| 1009 |
+
"batch_wall_latency_seconds": 1.2976085640257224,
|
| 1010 |
+
"amortized_generation_ms_per_sample": 1287.8015459282324,
|
| 1011 |
+
"amortized_batch_wall_ms_per_sample": 1297.6085640257224,
|
| 1012 |
+
"outputs": [
|
| 1013 |
+
{
|
| 1014 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[100,180,520,720],\"score\":0.95},{\"bbox_2d\":[450,0,990,740],\"score\":0.96}]}",
|
| 1015 |
+
"valid_prediction": {
|
| 1016 |
+
"objects": [
|
| 1017 |
+
{
|
| 1018 |
+
"bbox_2d": [
|
| 1019 |
+
100,
|
| 1020 |
+
180,
|
| 1021 |
+
520,
|
| 1022 |
+
720
|
| 1023 |
+
],
|
| 1024 |
+
"score": 0.95
|
| 1025 |
+
},
|
| 1026 |
+
{
|
| 1027 |
+
"bbox_2d": [
|
| 1028 |
+
450,
|
| 1029 |
+
0,
|
| 1030 |
+
990,
|
| 1031 |
+
740
|
| 1032 |
+
],
|
| 1033 |
+
"score": 0.96
|
| 1034 |
+
}
|
| 1035 |
+
]
|
| 1036 |
+
},
|
| 1037 |
+
"error": null,
|
| 1038 |
+
"finish_reason": "stop",
|
| 1039 |
+
"completion_tokens": 59,
|
| 1040 |
+
"prompt_tokens": 393
|
| 1041 |
+
}
|
| 1042 |
+
]
|
| 1043 |
+
},
|
| 1044 |
+
{
|
| 1045 |
+
"sample_ids": [
|
| 1046 |
+
"testA-43889-120792"
|
| 1047 |
+
],
|
| 1048 |
+
"batch_size": 1,
|
| 1049 |
+
"generation_latency_seconds": 0.7570710440631956,
|
| 1050 |
+
"batch_wall_latency_seconds": 0.7649242469342425,
|
| 1051 |
+
"amortized_generation_ms_per_sample": 757.0710440631956,
|
| 1052 |
+
"amortized_batch_wall_ms_per_sample": 764.9242469342425,
|
| 1053 |
+
"outputs": [
|
| 1054 |
+
{
|
| 1055 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[120,260,340,830],\"score\":0.95}]}",
|
| 1056 |
+
"valid_prediction": {
|
| 1057 |
+
"objects": [
|
| 1058 |
+
{
|
| 1059 |
+
"bbox_2d": [
|
| 1060 |
+
120,
|
| 1061 |
+
260,
|
| 1062 |
+
340,
|
| 1063 |
+
830
|
| 1064 |
+
],
|
| 1065 |
+
"score": 0.95
|
| 1066 |
+
}
|
| 1067 |
+
]
|
| 1068 |
+
},
|
| 1069 |
+
"error": null,
|
| 1070 |
+
"finish_reason": "stop",
|
| 1071 |
+
"completion_tokens": 33,
|
| 1072 |
+
"prompt_tokens": 531
|
| 1073 |
+
}
|
| 1074 |
+
]
|
| 1075 |
+
},
|
| 1076 |
+
{
|
| 1077 |
+
"sample_ids": [
|
| 1078 |
+
"testB-36082-99363"
|
| 1079 |
+
],
|
| 1080 |
+
"batch_size": 1,
|
| 1081 |
+
"generation_latency_seconds": 1.2985183690907434,
|
| 1082 |
+
"batch_wall_latency_seconds": 1.3064349760534242,
|
| 1083 |
+
"amortized_generation_ms_per_sample": 1298.5183690907434,
|
| 1084 |
+
"amortized_batch_wall_ms_per_sample": 1306.4349760534242,
|
| 1085 |
+
"outputs": [
|
| 1086 |
+
{
|
| 1087 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[0,100,420,890] ,\"score\":0.95},{\"bbox_2d\":[440,100,990,890] ,\"score\":0.95}]}",
|
| 1088 |
+
"valid_prediction": {
|
| 1089 |
+
"objects": [
|
| 1090 |
+
{
|
| 1091 |
+
"bbox_2d": [
|
| 1092 |
+
0,
|
| 1093 |
+
100,
|
| 1094 |
+
420,
|
| 1095 |
+
890
|
| 1096 |
+
],
|
| 1097 |
+
"score": 0.95
|
| 1098 |
+
},
|
| 1099 |
+
{
|
| 1100 |
+
"bbox_2d": [
|
| 1101 |
+
440,
|
| 1102 |
+
100,
|
| 1103 |
+
990,
|
| 1104 |
+
890
|
| 1105 |
+
],
|
| 1106 |
+
"score": 0.95
|
| 1107 |
+
}
|
| 1108 |
+
]
|
| 1109 |
+
},
|
| 1110 |
+
"error": null,
|
| 1111 |
+
"finish_reason": "stop",
|
| 1112 |
+
"completion_tokens": 61,
|
| 1113 |
+
"prompt_tokens": 392
|
| 1114 |
+
}
|
| 1115 |
+
]
|
| 1116 |
+
},
|
| 1117 |
+
{
|
| 1118 |
+
"sample_ids": [
|
| 1119 |
+
"testB-33108-91152"
|
| 1120 |
+
],
|
| 1121 |
+
"batch_size": 1,
|
| 1122 |
+
"generation_latency_seconds": 0.9395933509804308,
|
| 1123 |
+
"batch_wall_latency_seconds": 0.9509444029536098,
|
| 1124 |
+
"amortized_generation_ms_per_sample": 939.5933509804308,
|
| 1125 |
+
"amortized_batch_wall_ms_per_sample": 950.9444029536098,
|
| 1126 |
+
"outputs": [
|
| 1127 |
+
{
|
| 1128 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[0 ,270 ,390 ,450 ],\"score\":0.95}]}",
|
| 1129 |
+
"valid_prediction": {
|
| 1130 |
+
"objects": [
|
| 1131 |
+
{
|
| 1132 |
+
"bbox_2d": [
|
| 1133 |
+
0,
|
| 1134 |
+
270,
|
| 1135 |
+
390,
|
| 1136 |
+
450
|
| 1137 |
+
],
|
| 1138 |
+
"score": 0.95
|
| 1139 |
+
}
|
| 1140 |
+
]
|
| 1141 |
+
},
|
| 1142 |
+
"error": null,
|
| 1143 |
+
"finish_reason": "stop",
|
| 1144 |
+
"completion_tokens": 41,
|
| 1145 |
+
"prompt_tokens": 522
|
| 1146 |
+
}
|
| 1147 |
+
]
|
| 1148 |
+
},
|
| 1149 |
+
{
|
| 1150 |
+
"sample_ids": [
|
| 1151 |
+
"testA-26856-73830"
|
| 1152 |
+
],
|
| 1153 |
+
"batch_size": 1,
|
| 1154 |
+
"generation_latency_seconds": 10.264511390007101,
|
| 1155 |
+
"batch_wall_latency_seconds": 10.279962090076879,
|
| 1156 |
+
"amortized_generation_ms_per_sample": 10264.511390007101,
|
| 1157 |
+
"amortized_batch_wall_ms_per_sample": 10279.962090076879,
|
| 1158 |
+
"outputs": [
|
| 1159 |
+
{
|
| 1160 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[575 ",
|
| 1161 |
+
"valid_prediction": null,
|
| 1162 |
+
"error": "Expecting ',' delimiter: line 1 column 529 (char 528)",
|
| 1163 |
+
"finish_reason": "length",
|
| 1164 |
+
"completion_tokens": 512,
|
| 1165 |
+
"prompt_tokens": 522
|
| 1166 |
+
}
|
| 1167 |
+
]
|
| 1168 |
+
},
|
| 1169 |
+
{
|
| 1170 |
+
"sample_ids": [
|
| 1171 |
+
"testA-81271-223709"
|
| 1172 |
+
],
|
| 1173 |
+
"batch_size": 1,
|
| 1174 |
+
"generation_latency_seconds": 0.23597171693108976,
|
| 1175 |
+
"batch_wall_latency_seconds": 0.2577737799147144,
|
| 1176 |
+
"amortized_generation_ms_per_sample": 235.97171693108976,
|
| 1177 |
+
"amortized_batch_wall_ms_per_sample": 257.7737799147144,
|
| 1178 |
+
"outputs": [
|
| 1179 |
+
{
|
| 1180 |
+
"raw_response": "{\"objects\":[]}",
|
| 1181 |
+
"valid_prediction": {
|
| 1182 |
+
"objects": []
|
| 1183 |
+
},
|
| 1184 |
+
"error": null,
|
| 1185 |
+
"finish_reason": "stop",
|
| 1186 |
+
"completion_tokens": 6,
|
| 1187 |
+
"prompt_tokens": 520
|
| 1188 |
+
}
|
| 1189 |
+
]
|
| 1190 |
+
},
|
| 1191 |
+
{
|
| 1192 |
+
"sample_ids": [
|
| 1193 |
+
"testA-54554-150141"
|
| 1194 |
+
],
|
| 1195 |
+
"batch_size": 1,
|
| 1196 |
+
"generation_latency_seconds": 0.6926666980143636,
|
| 1197 |
+
"batch_wall_latency_seconds": 0.710583986947313,
|
| 1198 |
+
"amortized_generation_ms_per_sample": 692.6666980143636,
|
| 1199 |
+
"amortized_batch_wall_ms_per_sample": 710.583986947313,
|
| 1200 |
+
"outputs": [
|
| 1201 |
+
{
|
| 1202 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[0,0,1000,250] ,\"score\":0.95}]}",
|
| 1203 |
+
"valid_prediction": {
|
| 1204 |
+
"objects": [
|
| 1205 |
+
{
|
| 1206 |
+
"bbox_2d": [
|
| 1207 |
+
0,
|
| 1208 |
+
0,
|
| 1209 |
+
1000,
|
| 1210 |
+
250
|
| 1211 |
+
],
|
| 1212 |
+
"score": 0.95
|
| 1213 |
+
}
|
| 1214 |
+
]
|
| 1215 |
+
},
|
| 1216 |
+
"error": null,
|
| 1217 |
+
"finish_reason": "stop",
|
| 1218 |
+
"completion_tokens": 31,
|
| 1219 |
+
"prompt_tokens": 530
|
| 1220 |
+
}
|
| 1221 |
+
]
|
| 1222 |
+
},
|
| 1223 |
+
{
|
| 1224 |
+
"sample_ids": [
|
| 1225 |
+
"testB-38636-106481"
|
| 1226 |
+
],
|
| 1227 |
+
"batch_size": 1,
|
| 1228 |
+
"generation_latency_seconds": 9.935319383977912,
|
| 1229 |
+
"batch_wall_latency_seconds": 9.945194306899793,
|
| 1230 |
+
"amortized_generation_ms_per_sample": 9935.319383977912,
|
| 1231 |
+
"amortized_batch_wall_ms_per_sample": 9945.194306899793,
|
| 1232 |
+
"outputs": [
|
| 1233 |
+
{
|
| 1234 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[430 ",
|
| 1235 |
+
"valid_prediction": null,
|
| 1236 |
+
"error": "Expecting ',' delimiter: line 1 column 529 (char 528)",
|
| 1237 |
+
"finish_reason": "length",
|
| 1238 |
+
"completion_tokens": 512,
|
| 1239 |
+
"prompt_tokens": 397
|
| 1240 |
+
}
|
| 1241 |
+
]
|
| 1242 |
+
},
|
| 1243 |
+
{
|
| 1244 |
+
"sample_ids": [
|
| 1245 |
+
"testA-5734-15837"
|
| 1246 |
+
],
|
| 1247 |
+
"batch_size": 1,
|
| 1248 |
+
"generation_latency_seconds": 1.2923860140144825,
|
| 1249 |
+
"batch_wall_latency_seconds": 1.3085041570011526,
|
| 1250 |
+
"amortized_generation_ms_per_sample": 1292.3860140144825,
|
| 1251 |
+
"amortized_batch_wall_ms_per_sample": 1308.5041570011526,
|
| 1252 |
+
"outputs": [
|
| 1253 |
+
{
|
| 1254 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[0,190,310,740],\"score\":0.95},{\"bbox_2d\":[400,230,660,690],\"score\":0.93}]}",
|
| 1255 |
+
"valid_prediction": {
|
| 1256 |
+
"objects": [
|
| 1257 |
+
{
|
| 1258 |
+
"bbox_2d": [
|
| 1259 |
+
0,
|
| 1260 |
+
190,
|
| 1261 |
+
310,
|
| 1262 |
+
740
|
| 1263 |
+
],
|
| 1264 |
+
"score": 0.95
|
| 1265 |
+
},
|
| 1266 |
+
{
|
| 1267 |
+
"bbox_2d": [
|
| 1268 |
+
400,
|
| 1269 |
+
230,
|
| 1270 |
+
660,
|
| 1271 |
+
690
|
| 1272 |
+
],
|
| 1273 |
+
"score": 0.93
|
| 1274 |
+
}
|
| 1275 |
+
]
|
| 1276 |
+
},
|
| 1277 |
+
"error": null,
|
| 1278 |
+
"finish_reason": "stop",
|
| 1279 |
+
"completion_tokens": 59,
|
| 1280 |
+
"prompt_tokens": 525
|
| 1281 |
+
}
|
| 1282 |
+
]
|
| 1283 |
+
},
|
| 1284 |
+
{
|
| 1285 |
+
"sample_ids": [
|
| 1286 |
+
"testB-67558-186325"
|
| 1287 |
+
],
|
| 1288 |
+
"batch_size": 1,
|
| 1289 |
+
"generation_latency_seconds": 0.23707255406770855,
|
| 1290 |
+
"batch_wall_latency_seconds": 0.24415827100165188,
|
| 1291 |
+
"amortized_generation_ms_per_sample": 237.07255406770855,
|
| 1292 |
+
"amortized_batch_wall_ms_per_sample": 244.15827100165188,
|
| 1293 |
+
"outputs": [
|
| 1294 |
+
{
|
| 1295 |
+
"raw_response": "{\"objects\":[]}",
|
| 1296 |
+
"valid_prediction": {
|
| 1297 |
+
"objects": []
|
| 1298 |
+
},
|
| 1299 |
+
"error": null,
|
| 1300 |
+
"finish_reason": "stop",
|
| 1301 |
+
"completion_tokens": 5,
|
| 1302 |
+
"prompt_tokens": 520
|
| 1303 |
+
}
|
| 1304 |
+
]
|
| 1305 |
+
},
|
| 1306 |
+
{
|
| 1307 |
+
"sample_ids": [
|
| 1308 |
+
"testB-47053-129554"
|
| 1309 |
+
],
|
| 1310 |
+
"batch_size": 1,
|
| 1311 |
+
"generation_latency_seconds": 0.8615183549700305,
|
| 1312 |
+
"batch_wall_latency_seconds": 0.8810114429797977,
|
| 1313 |
+
"amortized_generation_ms_per_sample": 861.5183549700305,
|
| 1314 |
+
"amortized_batch_wall_ms_per_sample": 881.0114429797977,
|
| 1315 |
+
"outputs": [
|
| 1316 |
+
{
|
| 1317 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[420 ,300 ,800 ,800 ],\"score\":0.95}]}",
|
| 1318 |
+
"valid_prediction": {
|
| 1319 |
+
"objects": [
|
| 1320 |
+
{
|
| 1321 |
+
"bbox_2d": [
|
| 1322 |
+
420,
|
| 1323 |
+
300,
|
| 1324 |
+
800,
|
| 1325 |
+
800
|
| 1326 |
+
],
|
| 1327 |
+
"score": 0.95
|
| 1328 |
+
}
|
| 1329 |
+
]
|
| 1330 |
+
},
|
| 1331 |
+
"error": null,
|
| 1332 |
+
"finish_reason": "stop",
|
| 1333 |
+
"completion_tokens": 39,
|
| 1334 |
+
"prompt_tokens": 525
|
| 1335 |
+
}
|
| 1336 |
+
]
|
| 1337 |
+
},
|
| 1338 |
+
{
|
| 1339 |
+
"sample_ids": [
|
| 1340 |
+
"testA-93064-256422"
|
| 1341 |
+
],
|
| 1342 |
+
"batch_size": 1,
|
| 1343 |
+
"generation_latency_seconds": 0.7549799080006778,
|
| 1344 |
+
"batch_wall_latency_seconds": 0.7621725639328361,
|
| 1345 |
+
"amortized_generation_ms_per_sample": 754.9799080006778,
|
| 1346 |
+
"amortized_batch_wall_ms_per_sample": 762.172563932836,
|
| 1347 |
+
"outputs": [
|
| 1348 |
+
{
|
| 1349 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[100,80,750,980],\"score\":0.95}]}",
|
| 1350 |
+
"valid_prediction": {
|
| 1351 |
+
"objects": [
|
| 1352 |
+
{
|
| 1353 |
+
"bbox_2d": [
|
| 1354 |
+
100,
|
| 1355 |
+
80,
|
| 1356 |
+
750,
|
| 1357 |
+
980
|
| 1358 |
+
],
|
| 1359 |
+
"score": 0.95
|
| 1360 |
+
}
|
| 1361 |
+
]
|
| 1362 |
+
},
|
| 1363 |
+
"error": null,
|
| 1364 |
+
"finish_reason": "stop",
|
| 1365 |
+
"completion_tokens": 32,
|
| 1366 |
+
"prompt_tokens": 530
|
| 1367 |
+
}
|
| 1368 |
+
]
|
| 1369 |
+
},
|
| 1370 |
+
{
|
| 1371 |
+
"sample_ids": [
|
| 1372 |
+
"testB-60557-166821"
|
| 1373 |
+
],
|
| 1374 |
+
"batch_size": 1,
|
| 1375 |
+
"generation_latency_seconds": 0.23407667502760887,
|
| 1376 |
+
"batch_wall_latency_seconds": 0.24112907599192113,
|
| 1377 |
+
"amortized_generation_ms_per_sample": 234.07667502760887,
|
| 1378 |
+
"amortized_batch_wall_ms_per_sample": 241.12907599192113,
|
| 1379 |
+
"outputs": [
|
| 1380 |
+
{
|
| 1381 |
+
"raw_response": "{\"objects\":[]}",
|
| 1382 |
+
"valid_prediction": {
|
| 1383 |
+
"objects": []
|
| 1384 |
+
},
|
| 1385 |
+
"error": null,
|
| 1386 |
+
"finish_reason": "stop",
|
| 1387 |
+
"completion_tokens": 5,
|
| 1388 |
+
"prompt_tokens": 520
|
| 1389 |
+
}
|
| 1390 |
+
]
|
| 1391 |
+
},
|
| 1392 |
+
{
|
| 1393 |
+
"sample_ids": [
|
| 1394 |
+
"testA-55116-151688"
|
| 1395 |
+
],
|
| 1396 |
+
"batch_size": 1,
|
| 1397 |
+
"generation_latency_seconds": 0.22477965708822012,
|
| 1398 |
+
"batch_wall_latency_seconds": 0.23706394899636507,
|
| 1399 |
+
"amortized_generation_ms_per_sample": 224.77965708822012,
|
| 1400 |
+
"amortized_batch_wall_ms_per_sample": 237.06394899636507,
|
| 1401 |
+
"outputs": [
|
| 1402 |
+
{
|
| 1403 |
+
"raw_response": "{\"objects\":[]}",
|
| 1404 |
+
"valid_prediction": {
|
| 1405 |
+
"objects": []
|
| 1406 |
+
},
|
| 1407 |
+
"error": null,
|
| 1408 |
+
"finish_reason": "stop",
|
| 1409 |
+
"completion_tokens": 5,
|
| 1410 |
+
"prompt_tokens": 520
|
| 1411 |
+
}
|
| 1412 |
+
]
|
| 1413 |
+
},
|
| 1414 |
+
{
|
| 1415 |
+
"sample_ids": [
|
| 1416 |
+
"testB-65796-181392"
|
| 1417 |
+
],
|
| 1418 |
+
"batch_size": 1,
|
| 1419 |
+
"generation_latency_seconds": 0.24896689096931368,
|
| 1420 |
+
"batch_wall_latency_seconds": 0.26310552388895303,
|
| 1421 |
+
"amortized_generation_ms_per_sample": 248.96689096931368,
|
| 1422 |
+
"amortized_batch_wall_ms_per_sample": 263.10552388895303,
|
| 1423 |
+
"outputs": [
|
| 1424 |
+
{
|
| 1425 |
+
"raw_response": "{\"objects\":[]}",
|
| 1426 |
+
"valid_prediction": {
|
| 1427 |
+
"objects": []
|
| 1428 |
+
},
|
| 1429 |
+
"error": null,
|
| 1430 |
+
"finish_reason": "stop",
|
| 1431 |
+
"completion_tokens": 5,
|
| 1432 |
+
"prompt_tokens": 521
|
| 1433 |
+
}
|
| 1434 |
+
]
|
| 1435 |
+
},
|
| 1436 |
+
{
|
| 1437 |
+
"sample_ids": [
|
| 1438 |
+
"testB-23014-63278"
|
| 1439 |
+
],
|
| 1440 |
+
"batch_size": 1,
|
| 1441 |
+
"generation_latency_seconds": 0.8068696269765496,
|
| 1442 |
+
"batch_wall_latency_seconds": 0.8200291059911251,
|
| 1443 |
+
"amortized_generation_ms_per_sample": 806.8696269765496,
|
| 1444 |
+
"amortized_batch_wall_ms_per_sample": 820.0291059911251,
|
| 1445 |
+
"outputs": [
|
| 1446 |
+
{
|
| 1447 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[30,300,340,700],\"score\":0.95}]}",
|
| 1448 |
+
"valid_prediction": {
|
| 1449 |
+
"objects": [
|
| 1450 |
+
{
|
| 1451 |
+
"bbox_2d": [
|
| 1452 |
+
30,
|
| 1453 |
+
300,
|
| 1454 |
+
340,
|
| 1455 |
+
700
|
| 1456 |
+
],
|
| 1457 |
+
"score": 0.95
|
| 1458 |
+
}
|
| 1459 |
+
]
|
| 1460 |
+
},
|
| 1461 |
+
"error": null,
|
| 1462 |
+
"finish_reason": "stop",
|
| 1463 |
+
"completion_tokens": 32,
|
| 1464 |
+
"prompt_tokens": 523
|
| 1465 |
+
}
|
| 1466 |
+
]
|
| 1467 |
+
},
|
| 1468 |
+
{
|
| 1469 |
+
"sample_ids": [
|
| 1470 |
+
"testA-24562-67515"
|
| 1471 |
+
],
|
| 1472 |
+
"batch_size": 1,
|
| 1473 |
+
"generation_latency_seconds": 0.2276058099232614,
|
| 1474 |
+
"batch_wall_latency_seconds": 0.24499452603049576,
|
| 1475 |
+
"amortized_generation_ms_per_sample": 227.6058099232614,
|
| 1476 |
+
"amortized_batch_wall_ms_per_sample": 244.99452603049576,
|
| 1477 |
+
"outputs": [
|
| 1478 |
+
{
|
| 1479 |
+
"raw_response": "{\"objects\":[]}",
|
| 1480 |
+
"valid_prediction": {
|
| 1481 |
+
"objects": []
|
| 1482 |
+
},
|
| 1483 |
+
"error": null,
|
| 1484 |
+
"finish_reason": "stop",
|
| 1485 |
+
"completion_tokens": 5,
|
| 1486 |
+
"prompt_tokens": 526
|
| 1487 |
+
}
|
| 1488 |
+
]
|
| 1489 |
+
},
|
| 1490 |
+
{
|
| 1491 |
+
"sample_ids": [
|
| 1492 |
+
"testA-17736-48684"
|
| 1493 |
+
],
|
| 1494 |
+
"batch_size": 1,
|
| 1495 |
+
"generation_latency_seconds": 9.981450814986601,
|
| 1496 |
+
"batch_wall_latency_seconds": 9.995843482087366,
|
| 1497 |
+
"amortized_generation_ms_per_sample": 9981.450814986601,
|
| 1498 |
+
"amortized_batch_wall_ms_per_sample": 9995.843482087366,
|
| 1499 |
+
"outputs": [
|
| 1500 |
+
{
|
| 1501 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[500 \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n",
|
| 1502 |
+
"valid_prediction": null,
|
| 1503 |
+
"error": "Expecting ',' delimiter: line 499 column 1 (char 2020)",
|
| 1504 |
+
"finish_reason": "length",
|
| 1505 |
+
"completion_tokens": 512,
|
| 1506 |
+
"prompt_tokens": 525
|
| 1507 |
+
}
|
| 1508 |
+
]
|
| 1509 |
+
},
|
| 1510 |
+
{
|
| 1511 |
+
"sample_ids": [
|
| 1512 |
+
"testA-58475-160949"
|
| 1513 |
+
],
|
| 1514 |
+
"batch_size": 1,
|
| 1515 |
+
"generation_latency_seconds": 0.7565385290654376,
|
| 1516 |
+
"batch_wall_latency_seconds": 0.76229489792604,
|
| 1517 |
+
"amortized_generation_ms_per_sample": 756.5385290654376,
|
| 1518 |
+
"amortized_batch_wall_ms_per_sample": 762.29489792604,
|
| 1519 |
+
"outputs": [
|
| 1520 |
+
{
|
| 1521 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[40,10,580,980],\"score\":0.95}]}",
|
| 1522 |
+
"valid_prediction": {
|
| 1523 |
+
"objects": [
|
| 1524 |
+
{
|
| 1525 |
+
"bbox_2d": [
|
| 1526 |
+
40,
|
| 1527 |
+
10,
|
| 1528 |
+
580,
|
| 1529 |
+
980
|
| 1530 |
+
],
|
| 1531 |
+
"score": 0.95
|
| 1532 |
+
}
|
| 1533 |
+
]
|
| 1534 |
+
},
|
| 1535 |
+
"error": null,
|
| 1536 |
+
"finish_reason": "stop",
|
| 1537 |
+
"completion_tokens": 31,
|
| 1538 |
+
"prompt_tokens": 525
|
| 1539 |
+
}
|
| 1540 |
+
]
|
| 1541 |
+
},
|
| 1542 |
+
{
|
| 1543 |
+
"sample_ids": [
|
| 1544 |
+
"testA-80455-221493"
|
| 1545 |
+
],
|
| 1546 |
+
"batch_size": 1,
|
| 1547 |
+
"generation_latency_seconds": 1.0085041749989614,
|
| 1548 |
+
"batch_wall_latency_seconds": 1.020079139037989,
|
| 1549 |
+
"amortized_generation_ms_per_sample": 1008.5041749989614,
|
| 1550 |
+
"amortized_batch_wall_ms_per_sample": 1020.0791390379891,
|
| 1551 |
+
"outputs": [
|
| 1552 |
+
{
|
| 1553 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[0 , 0 , 560 , 890 ],\"score\":0.95}]}",
|
| 1554 |
+
"valid_prediction": {
|
| 1555 |
+
"objects": [
|
| 1556 |
+
{
|
| 1557 |
+
"bbox_2d": [
|
| 1558 |
+
0,
|
| 1559 |
+
0,
|
| 1560 |
+
560,
|
| 1561 |
+
890
|
| 1562 |
+
],
|
| 1563 |
+
"score": 0.95
|
| 1564 |
+
}
|
| 1565 |
+
]
|
| 1566 |
+
},
|
| 1567 |
+
"error": null,
|
| 1568 |
+
"finish_reason": "stop",
|
| 1569 |
+
"completion_tokens": 45,
|
| 1570 |
+
"prompt_tokens": 522
|
| 1571 |
+
}
|
| 1572 |
+
]
|
| 1573 |
+
},
|
| 1574 |
+
{
|
| 1575 |
+
"sample_ids": [
|
| 1576 |
+
"testA-31763-87405"
|
| 1577 |
+
],
|
| 1578 |
+
"batch_size": 1,
|
| 1579 |
+
"generation_latency_seconds": 1.3390386949758977,
|
| 1580 |
+
"batch_wall_latency_seconds": 1.3469314540270716,
|
| 1581 |
+
"amortized_generation_ms_per_sample": 1339.0386949758977,
|
| 1582 |
+
"amortized_batch_wall_ms_per_sample": 1346.9314540270716,
|
| 1583 |
+
"outputs": [
|
| 1584 |
+
{
|
| 1585 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[350,880,420,930] ,\"score\":0.92},{\"bbox_2d\":[580,750,640,810] ,\"score\":0.91}]}",
|
| 1586 |
+
"valid_prediction": {
|
| 1587 |
+
"objects": [
|
| 1588 |
+
{
|
| 1589 |
+
"bbox_2d": [
|
| 1590 |
+
350,
|
| 1591 |
+
880,
|
| 1592 |
+
420,
|
| 1593 |
+
930
|
| 1594 |
+
],
|
| 1595 |
+
"score": 0.92
|
| 1596 |
+
},
|
| 1597 |
+
{
|
| 1598 |
+
"bbox_2d": [
|
| 1599 |
+
580,
|
| 1600 |
+
750,
|
| 1601 |
+
640,
|
| 1602 |
+
810
|
| 1603 |
+
],
|
| 1604 |
+
"score": 0.91
|
| 1605 |
+
}
|
| 1606 |
+
]
|
| 1607 |
+
},
|
| 1608 |
+
"error": null,
|
| 1609 |
+
"finish_reason": "stop",
|
| 1610 |
+
"completion_tokens": 63,
|
| 1611 |
+
"prompt_tokens": 389
|
| 1612 |
+
}
|
| 1613 |
+
]
|
| 1614 |
+
},
|
| 1615 |
+
{
|
| 1616 |
+
"sample_ids": [
|
| 1617 |
+
"testB-23920-65742"
|
| 1618 |
+
],
|
| 1619 |
+
"batch_size": 1,
|
| 1620 |
+
"generation_latency_seconds": 10.071104103000835,
|
| 1621 |
+
"batch_wall_latency_seconds": 10.078898616950028,
|
| 1622 |
+
"amortized_generation_ms_per_sample": 10071.104103000835,
|
| 1623 |
+
"amortized_batch_wall_ms_per_sample": 10078.898616950028,
|
| 1624 |
+
"outputs": [
|
| 1625 |
+
{
|
| 1626 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[510 \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n",
|
| 1627 |
+
"valid_prediction": null,
|
| 1628 |
+
"error": "Expecting ',' delimiter: line 499 column 1 (char 2020)",
|
| 1629 |
+
"finish_reason": "length",
|
| 1630 |
+
"completion_tokens": 512,
|
| 1631 |
+
"prompt_tokens": 529
|
| 1632 |
+
}
|
| 1633 |
+
]
|
| 1634 |
+
},
|
| 1635 |
+
{
|
| 1636 |
+
"sample_ids": [
|
| 1637 |
+
"testA-469-1309"
|
| 1638 |
+
],
|
| 1639 |
+
"batch_size": 1,
|
| 1640 |
+
"generation_latency_seconds": 0.26207074394915253,
|
| 1641 |
+
"batch_wall_latency_seconds": 0.2735602979082614,
|
| 1642 |
+
"amortized_generation_ms_per_sample": 262.07074394915253,
|
| 1643 |
+
"amortized_batch_wall_ms_per_sample": 273.5602979082614,
|
| 1644 |
+
"outputs": [
|
| 1645 |
+
{
|
| 1646 |
+
"raw_response": "{\"objects\":[]}",
|
| 1647 |
+
"valid_prediction": {
|
| 1648 |
+
"objects": []
|
| 1649 |
+
},
|
| 1650 |
+
"error": null,
|
| 1651 |
+
"finish_reason": "stop",
|
| 1652 |
+
"completion_tokens": 5,
|
| 1653 |
+
"prompt_tokens": 522
|
| 1654 |
+
}
|
| 1655 |
+
]
|
| 1656 |
+
},
|
| 1657 |
+
{
|
| 1658 |
+
"sample_ids": [
|
| 1659 |
+
"testB-56691-155998"
|
| 1660 |
+
],
|
| 1661 |
+
"batch_size": 1,
|
| 1662 |
+
"generation_latency_seconds": 10.205810929066502,
|
| 1663 |
+
"batch_wall_latency_seconds": 10.223061772994697,
|
| 1664 |
+
"amortized_generation_ms_per_sample": 10205.810929066502,
|
| 1665 |
+
"amortized_batch_wall_ms_per_sample": 10223.061772994697,
|
| 1666 |
+
"outputs": [
|
| 1667 |
+
{
|
| 1668 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[690 ",
|
| 1669 |
+
"valid_prediction": null,
|
| 1670 |
+
"error": "Expecting ',' delimiter: line 1 column 529 (char 528)",
|
| 1671 |
+
"finish_reason": "length",
|
| 1672 |
+
"completion_tokens": 512,
|
| 1673 |
+
"prompt_tokens": 523
|
| 1674 |
+
}
|
| 1675 |
+
]
|
| 1676 |
+
},
|
| 1677 |
+
{
|
| 1678 |
+
"sample_ids": [
|
| 1679 |
+
"testB-31493-86668"
|
| 1680 |
+
],
|
| 1681 |
+
"batch_size": 1,
|
| 1682 |
+
"generation_latency_seconds": 10.434396203025244,
|
| 1683 |
+
"batch_wall_latency_seconds": 10.448493773932569,
|
| 1684 |
+
"amortized_generation_ms_per_sample": 10434.396203025244,
|
| 1685 |
+
"amortized_batch_wall_ms_per_sample": 10448.493773932569,
|
| 1686 |
+
"outputs": [
|
| 1687 |
+
{
|
| 1688 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[158 ",
|
| 1689 |
+
"valid_prediction": null,
|
| 1690 |
+
"error": "Expecting ',' delimiter: line 1 column 529 (char 528)",
|
| 1691 |
+
"finish_reason": "length",
|
| 1692 |
+
"completion_tokens": 512,
|
| 1693 |
+
"prompt_tokens": 535
|
| 1694 |
+
}
|
| 1695 |
+
]
|
| 1696 |
+
},
|
| 1697 |
+
{
|
| 1698 |
+
"sample_ids": [
|
| 1699 |
+
"testA-59284-163254"
|
| 1700 |
+
],
|
| 1701 |
+
"batch_size": 1,
|
| 1702 |
+
"generation_latency_seconds": 1.32271799701266,
|
| 1703 |
+
"batch_wall_latency_seconds": 1.3361050819512457,
|
| 1704 |
+
"amortized_generation_ms_per_sample": 1322.71799701266,
|
| 1705 |
+
"amortized_batch_wall_ms_per_sample": 1336.1050819512457,
|
| 1706 |
+
"outputs": [
|
| 1707 |
+
{
|
| 1708 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[490,360,680,970],\"score\":0.95},{\"bbox_2d\":[580,310,780,370],\"score\":0.92}]}",
|
| 1709 |
+
"valid_prediction": {
|
| 1710 |
+
"objects": [
|
| 1711 |
+
{
|
| 1712 |
+
"bbox_2d": [
|
| 1713 |
+
490,
|
| 1714 |
+
360,
|
| 1715 |
+
680,
|
| 1716 |
+
970
|
| 1717 |
+
],
|
| 1718 |
+
"score": 0.95
|
| 1719 |
+
},
|
| 1720 |
+
{
|
| 1721 |
+
"bbox_2d": [
|
| 1722 |
+
580,
|
| 1723 |
+
310,
|
| 1724 |
+
780,
|
| 1725 |
+
370
|
| 1726 |
+
],
|
| 1727 |
+
"score": 0.92
|
| 1728 |
+
}
|
| 1729 |
+
]
|
| 1730 |
+
},
|
| 1731 |
+
"error": null,
|
| 1732 |
+
"finish_reason": "stop",
|
| 1733 |
+
"completion_tokens": 61,
|
| 1734 |
+
"prompt_tokens": 543
|
| 1735 |
+
}
|
| 1736 |
+
]
|
| 1737 |
+
},
|
| 1738 |
+
{
|
| 1739 |
+
"sample_ids": [
|
| 1740 |
+
"testA-57397-158015"
|
| 1741 |
+
],
|
| 1742 |
+
"batch_size": 1,
|
| 1743 |
+
"generation_latency_seconds": 0.22141807794105262,
|
| 1744 |
+
"batch_wall_latency_seconds": 0.2377260560169816,
|
| 1745 |
+
"amortized_generation_ms_per_sample": 221.41807794105262,
|
| 1746 |
+
"amortized_batch_wall_ms_per_sample": 237.7260560169816,
|
| 1747 |
+
"outputs": [
|
| 1748 |
+
{
|
| 1749 |
+
"raw_response": "{\"objects\":[]}",
|
| 1750 |
+
"valid_prediction": {
|
| 1751 |
+
"objects": []
|
| 1752 |
+
},
|
| 1753 |
+
"error": null,
|
| 1754 |
+
"finish_reason": "stop",
|
| 1755 |
+
"completion_tokens": 5,
|
| 1756 |
+
"prompt_tokens": 522
|
| 1757 |
+
}
|
| 1758 |
+
]
|
| 1759 |
+
},
|
| 1760 |
+
{
|
| 1761 |
+
"sample_ids": [
|
| 1762 |
+
"testB-2935-8111"
|
| 1763 |
+
],
|
| 1764 |
+
"batch_size": 1,
|
| 1765 |
+
"generation_latency_seconds": 10.570877194055356,
|
| 1766 |
+
"batch_wall_latency_seconds": 10.580568464938551,
|
| 1767 |
+
"amortized_generation_ms_per_sample": 10570.877194055356,
|
| 1768 |
+
"amortized_batch_wall_ms_per_sample": 10580.568464938551,
|
| 1769 |
+
"outputs": [
|
| 1770 |
+
{
|
| 1771 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[380 ",
|
| 1772 |
+
"valid_prediction": null,
|
| 1773 |
+
"error": "Expecting ',' delimiter: line 1 column 529 (char 528)",
|
| 1774 |
+
"finish_reason": "length",
|
| 1775 |
+
"completion_tokens": 512,
|
| 1776 |
+
"prompt_tokens": 530
|
| 1777 |
+
}
|
| 1778 |
+
]
|
| 1779 |
+
},
|
| 1780 |
+
{
|
| 1781 |
+
"sample_ids": [
|
| 1782 |
+
"testB-22290-61226"
|
| 1783 |
+
],
|
| 1784 |
+
"batch_size": 1,
|
| 1785 |
+
"generation_latency_seconds": 0.22713651705998927,
|
| 1786 |
+
"batch_wall_latency_seconds": 0.23939726105891168,
|
| 1787 |
+
"amortized_generation_ms_per_sample": 227.13651705998927,
|
| 1788 |
+
"amortized_batch_wall_ms_per_sample": 239.39726105891168,
|
| 1789 |
+
"outputs": [
|
| 1790 |
+
{
|
| 1791 |
+
"raw_response": "{\"objects\":[]}",
|
| 1792 |
+
"valid_prediction": {
|
| 1793 |
+
"objects": []
|
| 1794 |
+
},
|
| 1795 |
+
"error": null,
|
| 1796 |
+
"finish_reason": "stop",
|
| 1797 |
+
"completion_tokens": 5,
|
| 1798 |
+
"prompt_tokens": 526
|
| 1799 |
+
}
|
| 1800 |
+
]
|
| 1801 |
+
},
|
| 1802 |
+
{
|
| 1803 |
+
"sample_ids": [
|
| 1804 |
+
"testB-8416-23140"
|
| 1805 |
+
],
|
| 1806 |
+
"batch_size": 1,
|
| 1807 |
+
"generation_latency_seconds": 1.8887132819509134,
|
| 1808 |
+
"batch_wall_latency_seconds": 1.9046463000122458,
|
| 1809 |
+
"amortized_generation_ms_per_sample": 1888.7132819509134,
|
| 1810 |
+
"amortized_batch_wall_ms_per_sample": 1904.6463000122458,
|
| 1811 |
+
"outputs": [
|
| 1812 |
+
{
|
| 1813 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[0,280,330,720] ,\"score\":0.95},{\"bbox_2d\":[170,270,560,720] ,\"score\":0.94},{\"bbox_2d\":[730,190,960,790] ,\"score\":0.93}]}",
|
| 1814 |
+
"valid_prediction": {
|
| 1815 |
+
"objects": [
|
| 1816 |
+
{
|
| 1817 |
+
"bbox_2d": [
|
| 1818 |
+
0,
|
| 1819 |
+
280,
|
| 1820 |
+
330,
|
| 1821 |
+
720
|
| 1822 |
+
],
|
| 1823 |
+
"score": 0.95
|
| 1824 |
+
},
|
| 1825 |
+
{
|
| 1826 |
+
"bbox_2d": [
|
| 1827 |
+
170,
|
| 1828 |
+
270,
|
| 1829 |
+
560,
|
| 1830 |
+
720
|
| 1831 |
+
],
|
| 1832 |
+
"score": 0.94
|
| 1833 |
+
},
|
| 1834 |
+
{
|
| 1835 |
+
"bbox_2d": [
|
| 1836 |
+
730,
|
| 1837 |
+
190,
|
| 1838 |
+
960,
|
| 1839 |
+
790
|
| 1840 |
+
],
|
| 1841 |
+
"score": 0.93
|
| 1842 |
+
}
|
| 1843 |
+
]
|
| 1844 |
+
},
|
| 1845 |
+
"error": null,
|
| 1846 |
+
"finish_reason": "stop",
|
| 1847 |
+
"completion_tokens": 90,
|
| 1848 |
+
"prompt_tokens": 525
|
| 1849 |
+
}
|
| 1850 |
+
]
|
| 1851 |
+
},
|
| 1852 |
+
{
|
| 1853 |
+
"sample_ids": [
|
| 1854 |
+
"testA-60731-167320"
|
| 1855 |
+
],
|
| 1856 |
+
"batch_size": 1,
|
| 1857 |
+
"generation_latency_seconds": 1.2317780090961605,
|
| 1858 |
+
"batch_wall_latency_seconds": 1.246546880924143,
|
| 1859 |
+
"amortized_generation_ms_per_sample": 1231.7780090961605,
|
| 1860 |
+
"amortized_batch_wall_ms_per_sample": 1246.546880924143,
|
| 1861 |
+
"outputs": [
|
| 1862 |
+
{
|
| 1863 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[0,80,420,910],\"score\":0.95},{\"bbox_2d\":[460,0,760,200],\"score\":0.85}]}",
|
| 1864 |
+
"valid_prediction": {
|
| 1865 |
+
"objects": [
|
| 1866 |
+
{
|
| 1867 |
+
"bbox_2d": [
|
| 1868 |
+
0,
|
| 1869 |
+
80,
|
| 1870 |
+
420,
|
| 1871 |
+
910
|
| 1872 |
+
],
|
| 1873 |
+
"score": 0.95
|
| 1874 |
+
},
|
| 1875 |
+
{
|
| 1876 |
+
"bbox_2d": [
|
| 1877 |
+
460,
|
| 1878 |
+
0,
|
| 1879 |
+
760,
|
| 1880 |
+
200
|
| 1881 |
+
],
|
| 1882 |
+
"score": 0.85
|
| 1883 |
+
}
|
| 1884 |
+
]
|
| 1885 |
+
},
|
| 1886 |
+
"error": null,
|
| 1887 |
+
"finish_reason": "stop",
|
| 1888 |
+
"completion_tokens": 56,
|
| 1889 |
+
"prompt_tokens": 526
|
| 1890 |
+
}
|
| 1891 |
+
]
|
| 1892 |
+
},
|
| 1893 |
+
{
|
| 1894 |
+
"sample_ids": [
|
| 1895 |
+
"testA-49055-135086"
|
| 1896 |
+
],
|
| 1897 |
+
"batch_size": 1,
|
| 1898 |
+
"generation_latency_seconds": 1.8898930319119245,
|
| 1899 |
+
"batch_wall_latency_seconds": 1.899978275061585,
|
| 1900 |
+
"amortized_generation_ms_per_sample": 1889.8930319119245,
|
| 1901 |
+
"amortized_batch_wall_ms_per_sample": 1899.978275061585,
|
| 1902 |
+
"outputs": [
|
| 1903 |
+
{
|
| 1904 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[120,140,320,980] ,\"score\":0.95},{\"bbox_2d\":[320,160,620,970] ,\"score\":0.94},{\"bbox_2d\":[620,250,860,980] ,\"score\":0.93}]}",
|
| 1905 |
+
"valid_prediction": {
|
| 1906 |
+
"objects": [
|
| 1907 |
+
{
|
| 1908 |
+
"bbox_2d": [
|
| 1909 |
+
120,
|
| 1910 |
+
140,
|
| 1911 |
+
320,
|
| 1912 |
+
980
|
| 1913 |
+
],
|
| 1914 |
+
"score": 0.95
|
| 1915 |
+
},
|
| 1916 |
+
{
|
| 1917 |
+
"bbox_2d": [
|
| 1918 |
+
320,
|
| 1919 |
+
160,
|
| 1920 |
+
620,
|
| 1921 |
+
970
|
| 1922 |
+
],
|
| 1923 |
+
"score": 0.94
|
| 1924 |
+
},
|
| 1925 |
+
{
|
| 1926 |
+
"bbox_2d": [
|
| 1927 |
+
620,
|
| 1928 |
+
250,
|
| 1929 |
+
860,
|
| 1930 |
+
980
|
| 1931 |
+
],
|
| 1932 |
+
"score": 0.93
|
| 1933 |
+
}
|
| 1934 |
+
]
|
| 1935 |
+
},
|
| 1936 |
+
"error": null,
|
| 1937 |
+
"finish_reason": "stop",
|
| 1938 |
+
"completion_tokens": 92,
|
| 1939 |
+
"prompt_tokens": 526
|
| 1940 |
+
}
|
| 1941 |
+
]
|
| 1942 |
+
},
|
| 1943 |
+
{
|
| 1944 |
+
"sample_ids": [
|
| 1945 |
+
"testA-7857-21644"
|
| 1946 |
+
],
|
| 1947 |
+
"batch_size": 1,
|
| 1948 |
+
"generation_latency_seconds": 9.8525026650168,
|
| 1949 |
+
"batch_wall_latency_seconds": 9.865253204014152,
|
| 1950 |
+
"amortized_generation_ms_per_sample": 9852.5026650168,
|
| 1951 |
+
"amortized_batch_wall_ms_per_sample": 9865.253204014152,
|
| 1952 |
+
"outputs": [
|
| 1953 |
+
{
|
| 1954 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[650 ",
|
| 1955 |
+
"valid_prediction": null,
|
| 1956 |
+
"error": "Expecting ',' delimiter: line 1 column 529 (char 528)",
|
| 1957 |
+
"finish_reason": "length",
|
| 1958 |
+
"completion_tokens": 512,
|
| 1959 |
+
"prompt_tokens": 527
|
| 1960 |
+
}
|
| 1961 |
+
]
|
| 1962 |
+
},
|
| 1963 |
+
{
|
| 1964 |
+
"sample_ids": [
|
| 1965 |
+
"testB-79645-219273"
|
| 1966 |
+
],
|
| 1967 |
+
"batch_size": 1,
|
| 1968 |
+
"generation_latency_seconds": 2.319699679967016,
|
| 1969 |
+
"batch_wall_latency_seconds": 2.332257404923439,
|
| 1970 |
+
"amortized_generation_ms_per_sample": 2319.699679967016,
|
| 1971 |
+
"amortized_batch_wall_ms_per_sample": 2332.257404923439,
|
| 1972 |
+
"outputs": [
|
| 1973 |
+
{
|
| 1974 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[120,420,540,700] ,\"score\":0.95},{\"bbox_2d\":[540,400,780,690] ,\"score\":0.94},{\"bbox_2d\":[0,450,120,670] ,\"score\":0.93},{\"bbox_2d\":[140,460,260,600] ,\"score\":0.92}]}",
|
| 1975 |
+
"valid_prediction": {
|
| 1976 |
+
"objects": [
|
| 1977 |
+
{
|
| 1978 |
+
"bbox_2d": [
|
| 1979 |
+
120,
|
| 1980 |
+
420,
|
| 1981 |
+
540,
|
| 1982 |
+
700
|
| 1983 |
+
],
|
| 1984 |
+
"score": 0.95
|
| 1985 |
+
},
|
| 1986 |
+
{
|
| 1987 |
+
"bbox_2d": [
|
| 1988 |
+
540,
|
| 1989 |
+
400,
|
| 1990 |
+
780,
|
| 1991 |
+
690
|
| 1992 |
+
],
|
| 1993 |
+
"score": 0.94
|
| 1994 |
+
},
|
| 1995 |
+
{
|
| 1996 |
+
"bbox_2d": [
|
| 1997 |
+
0,
|
| 1998 |
+
450,
|
| 1999 |
+
120,
|
| 2000 |
+
670
|
| 2001 |
+
],
|
| 2002 |
+
"score": 0.93
|
| 2003 |
+
},
|
| 2004 |
+
{
|
| 2005 |
+
"bbox_2d": [
|
| 2006 |
+
140,
|
| 2007 |
+
460,
|
| 2008 |
+
260,
|
| 2009 |
+
600
|
| 2010 |
+
],
|
| 2011 |
+
"score": 0.92
|
| 2012 |
+
}
|
| 2013 |
+
]
|
| 2014 |
+
},
|
| 2015 |
+
"error": null,
|
| 2016 |
+
"finish_reason": "stop",
|
| 2017 |
+
"completion_tokens": 119,
|
| 2018 |
+
"prompt_tokens": 521
|
| 2019 |
+
}
|
| 2020 |
+
]
|
| 2021 |
+
},
|
| 2022 |
+
{
|
| 2023 |
+
"sample_ids": [
|
| 2024 |
+
"testB-52921-145707"
|
| 2025 |
+
],
|
| 2026 |
+
"batch_size": 1,
|
| 2027 |
+
"generation_latency_seconds": 10.00177818897646,
|
| 2028 |
+
"batch_wall_latency_seconds": 10.020861182012595,
|
| 2029 |
+
"amortized_generation_ms_per_sample": 10001.77818897646,
|
| 2030 |
+
"amortized_batch_wall_ms_per_sample": 10020.861182012595,
|
| 2031 |
+
"outputs": [
|
| 2032 |
+
{
|
| 2033 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[380 ",
|
| 2034 |
+
"valid_prediction": null,
|
| 2035 |
+
"error": "Expecting ',' delimiter: line 1 column 529 (char 528)",
|
| 2036 |
+
"finish_reason": "length",
|
| 2037 |
+
"completion_tokens": 512,
|
| 2038 |
+
"prompt_tokens": 531
|
| 2039 |
+
}
|
| 2040 |
+
]
|
| 2041 |
+
},
|
| 2042 |
+
{
|
| 2043 |
+
"sample_ids": [
|
| 2044 |
+
"testB-69479-191562"
|
| 2045 |
+
],
|
| 2046 |
+
"batch_size": 1,
|
| 2047 |
+
"generation_latency_seconds": 0.6840293509885669,
|
| 2048 |
+
"batch_wall_latency_seconds": 0.70091468305327,
|
| 2049 |
+
"amortized_generation_ms_per_sample": 684.0293509885669,
|
| 2050 |
+
"amortized_batch_wall_ms_per_sample": 700.91468305327,
|
| 2051 |
+
"outputs": [
|
| 2052 |
+
{
|
| 2053 |
+
"raw_response": "{\"objects\":[{\"bbox_2d\":[0,30,974,986],\"score\":0.95}]}",
|
| 2054 |
+
"valid_prediction": {
|
| 2055 |
+
"objects": [
|
| 2056 |
+
{
|
| 2057 |
+
"bbox_2d": [
|
| 2058 |
+
0,
|
| 2059 |
+
30,
|
| 2060 |
+
974,
|
| 2061 |
+
986
|
| 2062 |
+
],
|
| 2063 |
+
"score": 0.95
|
| 2064 |
+
}
|
| 2065 |
+
]
|
| 2066 |
+
},
|
| 2067 |
+
"error": null,
|
| 2068 |
+
"finish_reason": "stop",
|
| 2069 |
+
"completion_tokens": 30,
|
| 2070 |
+
"prompt_tokens": 522
|
| 2071 |
+
}
|
| 2072 |
+
]
|
| 2073 |
+
},
|
| 2074 |
+
{
|
| 2075 |
+
"sample_ids": [
|
| 2076 |
+
"testA-69466-191524"
|
| 2077 |
+
],
|
| 2078 |
+
"batch_size": 1,
|
| 2079 |
+
"generation_latency_seconds": 0.24351457098964602,
|
| 2080 |
+
"batch_wall_latency_seconds": 0.2538871930446476,
|
| 2081 |
+
"amortized_generation_ms_per_sample": 243.51457098964602,
|
| 2082 |
+
"amortized_batch_wall_ms_per_sample": 253.88719304464757,
|
| 2083 |
+
"outputs": [
|
| 2084 |
+
{
|
| 2085 |
+
"raw_response": "{\"objects\":[]}",
|
| 2086 |
+
"valid_prediction": {
|
| 2087 |
+
"objects": []
|
| 2088 |
+
},
|
| 2089 |
+
"error": null,
|
| 2090 |
+
"finish_reason": "stop",
|
| 2091 |
+
"completion_tokens": 6,
|
| 2092 |
+
"prompt_tokens": 522
|
| 2093 |
+
}
|
| 2094 |
+
]
|
| 2095 |
+
}
|
| 2096 |
+
],
|
| 2097 |
+
"mean_generation_ms_per_sample": 3652.817064514238,
|
| 2098 |
+
"median_generation_ms_per_sample": 1280.612550501246,
|
| 2099 |
+
"mean_batch_wall_ms_per_sample": 3667.173011115665,
|
| 2100 |
+
"median_batch_wall_ms_per_sample": 1289.789535512682,
|
| 2101 |
+
"mean_prompt_tokens": 517.609375,
|
| 2102 |
+
"mean_completion_tokens": 178.0625,
|
| 2103 |
+
"latency_scope": "A800 vLLM-0.18.0 native generation latency with the frozen task prompt and structured-output schema; model loading and warmup excluded. Batch wall time includes the real request/image preparation path.",
|
| 2104 |
+
"flops_scope": "This file measures latency only. Counted FLOPs are measured separately by the HF reference probe and are not claimed to be complete vLLM kernel FLOPs."
|
| 2105 |
+
}
|
releases/20260913_lr/paper_data/main_table_cost_20260912/components/qwen_flops.json
ADDED
|
@@ -0,0 +1,341 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"model_kind": "Qwen3.8",
|
| 4 |
+
"task": "construction",
|
| 5 |
+
"model_path": "/mnt/HithinkOmniSSD/user_workspace/caisihang/checkpoints/Qwen3.8-27B-FP8",
|
| 6 |
+
"sample_count": 8,
|
| 7 |
+
"records": [
|
| 8 |
+
{
|
| 9 |
+
"sample_id": "0000184",
|
| 10 |
+
"prompt_tokens_in_reference_tensor": 1705,
|
| 11 |
+
"input_shapes": {
|
| 12 |
+
"input_ids": [
|
| 13 |
+
1,
|
| 14 |
+
1705
|
| 15 |
+
],
|
| 16 |
+
"attention_mask": [
|
| 17 |
+
1,
|
| 18 |
+
1705
|
| 19 |
+
],
|
| 20 |
+
"mm_token_type_ids": [
|
| 21 |
+
1,
|
| 22 |
+
1705
|
| 23 |
+
],
|
| 24 |
+
"pixel_values": [
|
| 25 |
+
3888,
|
| 26 |
+
1536
|
| 27 |
+
],
|
| 28 |
+
"image_grid_thw": [
|
| 29 |
+
1,
|
| 30 |
+
3
|
| 31 |
+
]
|
| 32 |
+
},
|
| 33 |
+
"prefill_flops_counted": 90083398713024,
|
| 34 |
+
"decode_one_token_flops_counted": 51922600128,
|
| 35 |
+
"prefill_gflops_counted": 90083.398713024,
|
| 36 |
+
"decode_one_token_gflops_counted": 51.922600128,
|
| 37 |
+
"prefill_operators": {
|
| 38 |
+
"aten.convolution": 20475543552,
|
| 39 |
+
"aten.addmm": 3283626885120,
|
| 40 |
+
"aten.bmm": 3741356981952,
|
| 41 |
+
"aten.mm": 83037939302400
|
| 42 |
+
},
|
| 43 |
+
"decode_operators": {
|
| 44 |
+
"aten.bmm": 670826688,
|
| 45 |
+
"aten.mm": 51243909120,
|
| 46 |
+
"aten.convolution": 7864320
|
| 47 |
+
}
|
| 48 |
+
},
|
| 49 |
+
{
|
| 50 |
+
"sample_id": "0001955",
|
| 51 |
+
"prompt_tokens_in_reference_tensor": 1705,
|
| 52 |
+
"input_shapes": {
|
| 53 |
+
"input_ids": [
|
| 54 |
+
1,
|
| 55 |
+
1705
|
| 56 |
+
],
|
| 57 |
+
"attention_mask": [
|
| 58 |
+
1,
|
| 59 |
+
1705
|
| 60 |
+
],
|
| 61 |
+
"mm_token_type_ids": [
|
| 62 |
+
1,
|
| 63 |
+
1705
|
| 64 |
+
],
|
| 65 |
+
"pixel_values": [
|
| 66 |
+
3888,
|
| 67 |
+
1536
|
| 68 |
+
],
|
| 69 |
+
"image_grid_thw": [
|
| 70 |
+
1,
|
| 71 |
+
3
|
| 72 |
+
]
|
| 73 |
+
},
|
| 74 |
+
"prefill_flops_counted": 90083398713024,
|
| 75 |
+
"decode_one_token_flops_counted": 51922600128,
|
| 76 |
+
"prefill_gflops_counted": 90083.398713024,
|
| 77 |
+
"decode_one_token_gflops_counted": 51.922600128,
|
| 78 |
+
"prefill_operators": {
|
| 79 |
+
"aten.convolution": 20475543552,
|
| 80 |
+
"aten.addmm": 3283626885120,
|
| 81 |
+
"aten.bmm": 3741356981952,
|
| 82 |
+
"aten.mm": 83037939302400
|
| 83 |
+
},
|
| 84 |
+
"decode_operators": {
|
| 85 |
+
"aten.bmm": 670826688,
|
| 86 |
+
"aten.mm": 51243909120,
|
| 87 |
+
"aten.convolution": 7864320
|
| 88 |
+
}
|
| 89 |
+
},
|
| 90 |
+
{
|
| 91 |
+
"sample_id": "0000840",
|
| 92 |
+
"prompt_tokens_in_reference_tensor": 1705,
|
| 93 |
+
"input_shapes": {
|
| 94 |
+
"input_ids": [
|
| 95 |
+
1,
|
| 96 |
+
1705
|
| 97 |
+
],
|
| 98 |
+
"attention_mask": [
|
| 99 |
+
1,
|
| 100 |
+
1705
|
| 101 |
+
],
|
| 102 |
+
"mm_token_type_ids": [
|
| 103 |
+
1,
|
| 104 |
+
1705
|
| 105 |
+
],
|
| 106 |
+
"pixel_values": [
|
| 107 |
+
3888,
|
| 108 |
+
1536
|
| 109 |
+
],
|
| 110 |
+
"image_grid_thw": [
|
| 111 |
+
1,
|
| 112 |
+
3
|
| 113 |
+
]
|
| 114 |
+
},
|
| 115 |
+
"prefill_flops_counted": 90083398713024,
|
| 116 |
+
"decode_one_token_flops_counted": 51922600128,
|
| 117 |
+
"prefill_gflops_counted": 90083.398713024,
|
| 118 |
+
"decode_one_token_gflops_counted": 51.922600128,
|
| 119 |
+
"prefill_operators": {
|
| 120 |
+
"aten.convolution": 20475543552,
|
| 121 |
+
"aten.addmm": 3283626885120,
|
| 122 |
+
"aten.bmm": 3741356981952,
|
| 123 |
+
"aten.mm": 83037939302400
|
| 124 |
+
},
|
| 125 |
+
"decode_operators": {
|
| 126 |
+
"aten.bmm": 670826688,
|
| 127 |
+
"aten.mm": 51243909120,
|
| 128 |
+
"aten.convolution": 7864320
|
| 129 |
+
}
|
| 130 |
+
},
|
| 131 |
+
{
|
| 132 |
+
"sample_id": "0003887",
|
| 133 |
+
"prompt_tokens_in_reference_tensor": 1705,
|
| 134 |
+
"input_shapes": {
|
| 135 |
+
"input_ids": [
|
| 136 |
+
1,
|
| 137 |
+
1705
|
| 138 |
+
],
|
| 139 |
+
"attention_mask": [
|
| 140 |
+
1,
|
| 141 |
+
1705
|
| 142 |
+
],
|
| 143 |
+
"mm_token_type_ids": [
|
| 144 |
+
1,
|
| 145 |
+
1705
|
| 146 |
+
],
|
| 147 |
+
"pixel_values": [
|
| 148 |
+
3888,
|
| 149 |
+
1536
|
| 150 |
+
],
|
| 151 |
+
"image_grid_thw": [
|
| 152 |
+
1,
|
| 153 |
+
3
|
| 154 |
+
]
|
| 155 |
+
},
|
| 156 |
+
"prefill_flops_counted": 90083398713024,
|
| 157 |
+
"decode_one_token_flops_counted": 51922600128,
|
| 158 |
+
"prefill_gflops_counted": 90083.398713024,
|
| 159 |
+
"decode_one_token_gflops_counted": 51.922600128,
|
| 160 |
+
"prefill_operators": {
|
| 161 |
+
"aten.convolution": 20475543552,
|
| 162 |
+
"aten.addmm": 3283626885120,
|
| 163 |
+
"aten.bmm": 3741356981952,
|
| 164 |
+
"aten.mm": 83037939302400
|
| 165 |
+
},
|
| 166 |
+
"decode_operators": {
|
| 167 |
+
"aten.bmm": 670826688,
|
| 168 |
+
"aten.mm": 51243909120,
|
| 169 |
+
"aten.convolution": 7864320
|
| 170 |
+
}
|
| 171 |
+
},
|
| 172 |
+
{
|
| 173 |
+
"sample_id": "0003433",
|
| 174 |
+
"prompt_tokens_in_reference_tensor": 1705,
|
| 175 |
+
"input_shapes": {
|
| 176 |
+
"input_ids": [
|
| 177 |
+
1,
|
| 178 |
+
1705
|
| 179 |
+
],
|
| 180 |
+
"attention_mask": [
|
| 181 |
+
1,
|
| 182 |
+
1705
|
| 183 |
+
],
|
| 184 |
+
"mm_token_type_ids": [
|
| 185 |
+
1,
|
| 186 |
+
1705
|
| 187 |
+
],
|
| 188 |
+
"pixel_values": [
|
| 189 |
+
3888,
|
| 190 |
+
1536
|
| 191 |
+
],
|
| 192 |
+
"image_grid_thw": [
|
| 193 |
+
1,
|
| 194 |
+
3
|
| 195 |
+
]
|
| 196 |
+
},
|
| 197 |
+
"prefill_flops_counted": 90083398713024,
|
| 198 |
+
"decode_one_token_flops_counted": 51922600128,
|
| 199 |
+
"prefill_gflops_counted": 90083.398713024,
|
| 200 |
+
"decode_one_token_gflops_counted": 51.922600128,
|
| 201 |
+
"prefill_operators": {
|
| 202 |
+
"aten.convolution": 20475543552,
|
| 203 |
+
"aten.addmm": 3283626885120,
|
| 204 |
+
"aten.bmm": 3741356981952,
|
| 205 |
+
"aten.mm": 83037939302400
|
| 206 |
+
},
|
| 207 |
+
"decode_operators": {
|
| 208 |
+
"aten.bmm": 670826688,
|
| 209 |
+
"aten.mm": 51243909120,
|
| 210 |
+
"aten.convolution": 7864320
|
| 211 |
+
}
|
| 212 |
+
},
|
| 213 |
+
{
|
| 214 |
+
"sample_id": "0001613",
|
| 215 |
+
"prompt_tokens_in_reference_tensor": 1705,
|
| 216 |
+
"input_shapes": {
|
| 217 |
+
"input_ids": [
|
| 218 |
+
1,
|
| 219 |
+
1705
|
| 220 |
+
],
|
| 221 |
+
"attention_mask": [
|
| 222 |
+
1,
|
| 223 |
+
1705
|
| 224 |
+
],
|
| 225 |
+
"mm_token_type_ids": [
|
| 226 |
+
1,
|
| 227 |
+
1705
|
| 228 |
+
],
|
| 229 |
+
"pixel_values": [
|
| 230 |
+
3888,
|
| 231 |
+
1536
|
| 232 |
+
],
|
| 233 |
+
"image_grid_thw": [
|
| 234 |
+
1,
|
| 235 |
+
3
|
| 236 |
+
]
|
| 237 |
+
},
|
| 238 |
+
"prefill_flops_counted": 90083398713024,
|
| 239 |
+
"decode_one_token_flops_counted": 51922600128,
|
| 240 |
+
"prefill_gflops_counted": 90083.398713024,
|
| 241 |
+
"decode_one_token_gflops_counted": 51.922600128,
|
| 242 |
+
"prefill_operators": {
|
| 243 |
+
"aten.convolution": 20475543552,
|
| 244 |
+
"aten.addmm": 3283626885120,
|
| 245 |
+
"aten.bmm": 3741356981952,
|
| 246 |
+
"aten.mm": 83037939302400
|
| 247 |
+
},
|
| 248 |
+
"decode_operators": {
|
| 249 |
+
"aten.bmm": 670826688,
|
| 250 |
+
"aten.mm": 51243909120,
|
| 251 |
+
"aten.convolution": 7864320
|
| 252 |
+
}
|
| 253 |
+
},
|
| 254 |
+
{
|
| 255 |
+
"sample_id": "0002591",
|
| 256 |
+
"prompt_tokens_in_reference_tensor": 1705,
|
| 257 |
+
"input_shapes": {
|
| 258 |
+
"input_ids": [
|
| 259 |
+
1,
|
| 260 |
+
1705
|
| 261 |
+
],
|
| 262 |
+
"attention_mask": [
|
| 263 |
+
1,
|
| 264 |
+
1705
|
| 265 |
+
],
|
| 266 |
+
"mm_token_type_ids": [
|
| 267 |
+
1,
|
| 268 |
+
1705
|
| 269 |
+
],
|
| 270 |
+
"pixel_values": [
|
| 271 |
+
3888,
|
| 272 |
+
1536
|
| 273 |
+
],
|
| 274 |
+
"image_grid_thw": [
|
| 275 |
+
1,
|
| 276 |
+
3
|
| 277 |
+
]
|
| 278 |
+
},
|
| 279 |
+
"prefill_flops_counted": 90083398713024,
|
| 280 |
+
"decode_one_token_flops_counted": 51922600128,
|
| 281 |
+
"prefill_gflops_counted": 90083.398713024,
|
| 282 |
+
"decode_one_token_gflops_counted": 51.922600128,
|
| 283 |
+
"prefill_operators": {
|
| 284 |
+
"aten.convolution": 20475543552,
|
| 285 |
+
"aten.addmm": 3283626885120,
|
| 286 |
+
"aten.bmm": 3741356981952,
|
| 287 |
+
"aten.mm": 83037939302400
|
| 288 |
+
},
|
| 289 |
+
"decode_operators": {
|
| 290 |
+
"aten.bmm": 670826688,
|
| 291 |
+
"aten.mm": 51243909120,
|
| 292 |
+
"aten.convolution": 7864320
|
| 293 |
+
}
|
| 294 |
+
},
|
| 295 |
+
{
|
| 296 |
+
"sample_id": "0003105",
|
| 297 |
+
"prompt_tokens_in_reference_tensor": 1705,
|
| 298 |
+
"input_shapes": {
|
| 299 |
+
"input_ids": [
|
| 300 |
+
1,
|
| 301 |
+
1705
|
| 302 |
+
],
|
| 303 |
+
"attention_mask": [
|
| 304 |
+
1,
|
| 305 |
+
1705
|
| 306 |
+
],
|
| 307 |
+
"mm_token_type_ids": [
|
| 308 |
+
1,
|
| 309 |
+
1705
|
| 310 |
+
],
|
| 311 |
+
"pixel_values": [
|
| 312 |
+
3888,
|
| 313 |
+
1536
|
| 314 |
+
],
|
| 315 |
+
"image_grid_thw": [
|
| 316 |
+
1,
|
| 317 |
+
3
|
| 318 |
+
]
|
| 319 |
+
},
|
| 320 |
+
"prefill_flops_counted": 90083398713024,
|
| 321 |
+
"decode_one_token_flops_counted": 51922600128,
|
| 322 |
+
"prefill_gflops_counted": 90083.398713024,
|
| 323 |
+
"decode_one_token_gflops_counted": 51.922600128,
|
| 324 |
+
"prefill_operators": {
|
| 325 |
+
"aten.convolution": 20475543552,
|
| 326 |
+
"aten.addmm": 3283626885120,
|
| 327 |
+
"aten.bmm": 3741356981952,
|
| 328 |
+
"aten.mm": 83037939302400
|
| 329 |
+
},
|
| 330 |
+
"decode_operators": {
|
| 331 |
+
"aten.bmm": 670826688,
|
| 332 |
+
"aten.mm": 51243909120,
|
| 333 |
+
"aten.convolution": 7864320
|
| 334 |
+
}
|
| 335 |
+
}
|
| 336 |
+
],
|
| 337 |
+
"mean_prefill_gflops_counted": 90083.398713024,
|
| 338 |
+
"mean_decode_one_token_gflops_counted": 51.922600128,
|
| 339 |
+
"scope": "HF reference execution on the A800 with real task inputs; dense products/convolutions/supported attention counted with multiply-add=2 FLOPs. vLLM custom kernels, elementwise/normalization/nonlinearities and other unsupported operations are not complete.",
|
| 340 |
+
"answer_generation": "not performed; one known-token decode probe only"
|
| 341 |
+
}
|
releases/20260913_lr/paper_data/main_table_cost_20260912/components/qwen_latency.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
releases/20260913_lr/paper_data/main_table_cost_20260912/components/yolo26x_router.json
ADDED
|
@@ -0,0 +1,238 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "complete",
|
| 3 |
+
"run": "shared_12_yolo26x_Qwen3.8",
|
| 4 |
+
"config": {
|
| 5 |
+
"fraction": 1.0,
|
| 6 |
+
"seed": 42,
|
| 7 |
+
"freeze_encoder": true,
|
| 8 |
+
"fusion": "concat",
|
| 9 |
+
"harm_cost": 1.0,
|
| 10 |
+
"batch_size": 256,
|
| 11 |
+
"weight_decay": 0.01,
|
| 12 |
+
"max_epochs": 50,
|
| 13 |
+
"patience": 8,
|
| 14 |
+
"cpu_threads": 1,
|
| 15 |
+
"student": "8M",
|
| 16 |
+
"representation_dim": 128,
|
| 17 |
+
"lr": 0.003,
|
| 18 |
+
"target": "four_state",
|
| 19 |
+
"strategy": "natural",
|
| 20 |
+
"hidden_percent": 90,
|
| 21 |
+
"expert": "yolo26x",
|
| 22 |
+
"task": "construction",
|
| 23 |
+
"endpoint": "Qwen3.8",
|
| 24 |
+
"branches": [
|
| 25 |
+
"image",
|
| 26 |
+
"hidden",
|
| 27 |
+
"confidence",
|
| 28 |
+
"output"
|
| 29 |
+
],
|
| 30 |
+
"hidden_layer": "feature_blocks__layer_16",
|
| 31 |
+
"id": "shared_12_yolo26x_Qwen3.8",
|
| 32 |
+
"stage": "shared_mandatory_inputs"
|
| 33 |
+
},
|
| 34 |
+
"checkpoint": "/mnt/HithinkOmniSSD/user_workspace/caisihang/research/ICASSP_ModelCollaboration/outputs/router_shared_image_hidden_20260910/runs/shared_12_yolo26x_Qwen3.8/best.pt",
|
| 35 |
+
"score": "logit_rescue-logit_harm",
|
| 36 |
+
"test_rows_recomputed": 3004,
|
| 37 |
+
"selected_decisions_match_main": true,
|
| 38 |
+
"max_score_abs_difference": 0.0,
|
| 39 |
+
"actual_call_fraction": 0.4876830892143808,
|
| 40 |
+
"sample_count": 64,
|
| 41 |
+
"sample_ids": [
|
| 42 |
+
"0002628",
|
| 43 |
+
"0003242",
|
| 44 |
+
"0001568",
|
| 45 |
+
"0001119",
|
| 46 |
+
"0003648",
|
| 47 |
+
"0000187",
|
| 48 |
+
"0001891",
|
| 49 |
+
"0001689",
|
| 50 |
+
"0003372",
|
| 51 |
+
"0002549",
|
| 52 |
+
"0000915",
|
| 53 |
+
"0003633",
|
| 54 |
+
"0004424",
|
| 55 |
+
"0001627",
|
| 56 |
+
"0001562",
|
| 57 |
+
"0002018",
|
| 58 |
+
"0000254",
|
| 59 |
+
"0002719",
|
| 60 |
+
"0002730",
|
| 61 |
+
"0004413",
|
| 62 |
+
"0000891",
|
| 63 |
+
"0000150",
|
| 64 |
+
"0003223",
|
| 65 |
+
"0004731",
|
| 66 |
+
"0001587",
|
| 67 |
+
"0000483",
|
| 68 |
+
"0001116",
|
| 69 |
+
"0003376",
|
| 70 |
+
"0001225",
|
| 71 |
+
"0003082",
|
| 72 |
+
"0004447",
|
| 73 |
+
"0000283",
|
| 74 |
+
"0000563",
|
| 75 |
+
"0001254",
|
| 76 |
+
"0004759",
|
| 77 |
+
"0001868",
|
| 78 |
+
"0001860",
|
| 79 |
+
"0004087",
|
| 80 |
+
"0002962",
|
| 81 |
+
"0001802",
|
| 82 |
+
"0002973",
|
| 83 |
+
"0002648",
|
| 84 |
+
"0004350",
|
| 85 |
+
"0004377",
|
| 86 |
+
"0004809",
|
| 87 |
+
"0003694",
|
| 88 |
+
"0001572",
|
| 89 |
+
"0001550",
|
| 90 |
+
"0000213",
|
| 91 |
+
"0002266",
|
| 92 |
+
"0002445",
|
| 93 |
+
"0004513",
|
| 94 |
+
"0002124",
|
| 95 |
+
"0004391",
|
| 96 |
+
"0003679",
|
| 97 |
+
"0002902",
|
| 98 |
+
"0002508",
|
| 99 |
+
"0004767",
|
| 100 |
+
"0003840",
|
| 101 |
+
"0003498",
|
| 102 |
+
"0001579",
|
| 103 |
+
"0001748",
|
| 104 |
+
"0004706",
|
| 105 |
+
"0004387"
|
| 106 |
+
],
|
| 107 |
+
"batch_size": 1,
|
| 108 |
+
"latency_ms": [
|
| 109 |
+
17.35539303626865,
|
| 110 |
+
13.579263002611697,
|
| 111 |
+
15.319477999582887,
|
| 112 |
+
14.879688038490713,
|
| 113 |
+
24.25052202306688,
|
| 114 |
+
20.667891018092632,
|
| 115 |
+
20.015343907289207,
|
| 116 |
+
19.221558002755046,
|
| 117 |
+
20.937222987413406,
|
| 118 |
+
26.26270893961191,
|
| 119 |
+
17.899635946378112,
|
| 120 |
+
16.57471200451255,
|
| 121 |
+
18.844021018594503,
|
| 122 |
+
21.3212959934026,
|
| 123 |
+
19.545868039131165,
|
| 124 |
+
64.57463896367699,
|
| 125 |
+
20.962884067557752,
|
| 126 |
+
24.004184058867395,
|
| 127 |
+
20.19065897911787,
|
| 128 |
+
18.50342599209398,
|
| 129 |
+
19.576229038648307,
|
| 130 |
+
25.253725005313754,
|
| 131 |
+
21.430211956612766,
|
| 132 |
+
25.169604923576117,
|
| 133 |
+
17.8982320940122,
|
| 134 |
+
21.415626979433,
|
| 135 |
+
22.9903319850564,
|
| 136 |
+
229.59673800505698,
|
| 137 |
+
19.89229000173509,
|
| 138 |
+
18.98771501146257,
|
| 139 |
+
18.157949903979897,
|
| 140 |
+
22.423187969252467,
|
| 141 |
+
19.864327972754836,
|
| 142 |
+
21.502194926142693,
|
| 143 |
+
18.544791964814067,
|
| 144 |
+
50.5554600385949,
|
| 145 |
+
20.046600024215877,
|
| 146 |
+
16.299219918437302,
|
| 147 |
+
18.668118049390614,
|
| 148 |
+
24.017272982746363,
|
| 149 |
+
24.73958197515458,
|
| 150 |
+
18.68593308608979,
|
| 151 |
+
20.250934991054237,
|
| 152 |
+
23.676551994867623,
|
| 153 |
+
23.003229987807572,
|
| 154 |
+
24.6098650386557,
|
| 155 |
+
21.308812079951167,
|
| 156 |
+
17.95686304103583,
|
| 157 |
+
18.23423698078841,
|
| 158 |
+
14.343310962431133,
|
| 159 |
+
25.8766250917688,
|
| 160 |
+
18.20633898023516,
|
| 161 |
+
30.77617404051125,
|
| 162 |
+
19.3530690157786,
|
| 163 |
+
15.836970997042954,
|
| 164 |
+
19.045730936340988,
|
| 165 |
+
16.162591986358166,
|
| 166 |
+
25.458221905864775,
|
| 167 |
+
18.277363968081772,
|
| 168 |
+
52.543785073794425,
|
| 169 |
+
22.9536690749228,
|
| 170 |
+
17.81571004539728,
|
| 171 |
+
18.550690030679107,
|
| 172 |
+
21.854626946151257
|
| 173 |
+
],
|
| 174 |
+
"mean_latency_ms_per_input": 25.261267359383055,
|
| 175 |
+
"head_mean_latency_ms": 0.486134999897331,
|
| 176 |
+
"gflops_per_input": [
|
| 177 |
+
3.183754752,
|
| 178 |
+
3.183754752,
|
| 179 |
+
3.183754752,
|
| 180 |
+
3.183754752,
|
| 181 |
+
3.183754752,
|
| 182 |
+
3.183754752,
|
| 183 |
+
3.183754752,
|
| 184 |
+
3.183754752
|
| 185 |
+
],
|
| 186 |
+
"mean_gflops_per_input": 3.183754752,
|
| 187 |
+
"operators": [
|
| 188 |
+
{
|
| 189 |
+
"aten.convolution": 228802560,
|
| 190 |
+
"aten.bmm": 679257600,
|
| 191 |
+
"aten._scaled_dot_product_flash_attention": 236027904,
|
| 192 |
+
"aten.addmm": 2039666688
|
| 193 |
+
},
|
| 194 |
+
{
|
| 195 |
+
"aten.convolution": 228802560,
|
| 196 |
+
"aten.bmm": 679257600,
|
| 197 |
+
"aten._scaled_dot_product_flash_attention": 236027904,
|
| 198 |
+
"aten.addmm": 2039666688
|
| 199 |
+
},
|
| 200 |
+
{
|
| 201 |
+
"aten.convolution": 228802560,
|
| 202 |
+
"aten.bmm": 679257600,
|
| 203 |
+
"aten._scaled_dot_product_flash_attention": 236027904,
|
| 204 |
+
"aten.addmm": 2039666688
|
| 205 |
+
},
|
| 206 |
+
{
|
| 207 |
+
"aten.convolution": 228802560,
|
| 208 |
+
"aten.bmm": 679257600,
|
| 209 |
+
"aten._scaled_dot_product_flash_attention": 236027904,
|
| 210 |
+
"aten.addmm": 2039666688
|
| 211 |
+
},
|
| 212 |
+
{
|
| 213 |
+
"aten.convolution": 228802560,
|
| 214 |
+
"aten.bmm": 679257600,
|
| 215 |
+
"aten._scaled_dot_product_flash_attention": 236027904,
|
| 216 |
+
"aten.addmm": 2039666688
|
| 217 |
+
},
|
| 218 |
+
{
|
| 219 |
+
"aten.convolution": 228802560,
|
| 220 |
+
"aten.bmm": 679257600,
|
| 221 |
+
"aten._scaled_dot_product_flash_attention": 236027904,
|
| 222 |
+
"aten.addmm": 2039666688
|
| 223 |
+
},
|
| 224 |
+
{
|
| 225 |
+
"aten.convolution": 228802560,
|
| 226 |
+
"aten.bmm": 679257600,
|
| 227 |
+
"aten._scaled_dot_product_flash_attention": 236027904,
|
| 228 |
+
"aten.addmm": 2039666688
|
| 229 |
+
},
|
| 230 |
+
{
|
| 231 |
+
"aten.convolution": 228802560,
|
| 232 |
+
"aten.bmm": 679257600,
|
| 233 |
+
"aten._scaled_dot_product_flash_attention": 236027904,
|
| 234 |
+
"aten.addmm": 2039666688
|
| 235 |
+
}
|
| 236 |
+
],
|
| 237 |
+
"scope": "image loading, bilinear resize, tokenize when used, two image slots (including gray placeholder), active distilled encoders, feature assembly and original routing head; cached specialist features; supported FLOPs only"
|
| 238 |
+
}
|
releases/20260913_lr/paper_data/main_table_cost_20260912/components/yolo26x_specialist.json
ADDED
|
@@ -0,0 +1,254 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"component": "specialist",
|
| 3 |
+
"expert": "yolo26x",
|
| 4 |
+
"task": "construction",
|
| 5 |
+
"device": "cuda:0",
|
| 6 |
+
"checkpoint": "/mnt/HithinkOmniSSD/user_workspace/caisihang/research/ICASSP_ModelCollaboration/models/construction/yolo26x.pt",
|
| 7 |
+
"sample_count": 64,
|
| 8 |
+
"batch_size": 1,
|
| 9 |
+
"sample_ids": [
|
| 10 |
+
"0000944",
|
| 11 |
+
"0003511",
|
| 12 |
+
"0003365",
|
| 13 |
+
"0001351",
|
| 14 |
+
"0002149",
|
| 15 |
+
"0003368",
|
| 16 |
+
"0000176",
|
| 17 |
+
"0004487",
|
| 18 |
+
"0000013",
|
| 19 |
+
"0003583",
|
| 20 |
+
"0000758",
|
| 21 |
+
"0001291",
|
| 22 |
+
"0001451",
|
| 23 |
+
"0002755",
|
| 24 |
+
"0003088",
|
| 25 |
+
"0004538",
|
| 26 |
+
"0004324",
|
| 27 |
+
"0000516",
|
| 28 |
+
"0003893",
|
| 29 |
+
"0003475",
|
| 30 |
+
"0002137",
|
| 31 |
+
"0003272",
|
| 32 |
+
"0001711",
|
| 33 |
+
"0003262",
|
| 34 |
+
"0001470",
|
| 35 |
+
"0002145",
|
| 36 |
+
"0002662",
|
| 37 |
+
"0000756",
|
| 38 |
+
"0001631",
|
| 39 |
+
"0001924",
|
| 40 |
+
"0002496",
|
| 41 |
+
"0001443",
|
| 42 |
+
"0004053",
|
| 43 |
+
"0002168",
|
| 44 |
+
"0001881",
|
| 45 |
+
"0001666",
|
| 46 |
+
"0000304",
|
| 47 |
+
"0001252",
|
| 48 |
+
"0004844",
|
| 49 |
+
"0000949",
|
| 50 |
+
"0003430",
|
| 51 |
+
"0003807",
|
| 52 |
+
"0004199",
|
| 53 |
+
"0002538",
|
| 54 |
+
"0002091",
|
| 55 |
+
"0001569",
|
| 56 |
+
"0003048",
|
| 57 |
+
"0002854",
|
| 58 |
+
"0000380",
|
| 59 |
+
"0001234",
|
| 60 |
+
"0001484",
|
| 61 |
+
"0003180",
|
| 62 |
+
"0000824",
|
| 63 |
+
"0003784",
|
| 64 |
+
"0000894",
|
| 65 |
+
"0004168",
|
| 66 |
+
"0000564",
|
| 67 |
+
"0004296",
|
| 68 |
+
"0003054",
|
| 69 |
+
"0001583",
|
| 70 |
+
"0003386",
|
| 71 |
+
"0000205",
|
| 72 |
+
"0004214",
|
| 73 |
+
"0003625"
|
| 74 |
+
],
|
| 75 |
+
"model_load_seconds_excluded_from_latency": true,
|
| 76 |
+
"forward_latency_ms_per_sample": [
|
| 77 |
+
41.741790771484375,
|
| 78 |
+
51.50377655029297,
|
| 79 |
+
78.82457733154297,
|
| 80 |
+
87.29910278320312,
|
| 81 |
+
49.47385787963867,
|
| 82 |
+
49.40816116333008,
|
| 83 |
+
168.38259887695312,
|
| 84 |
+
42.57542419433594,
|
| 85 |
+
40.342529296875,
|
| 86 |
+
54.95475387573242,
|
| 87 |
+
48.86323165893555,
|
| 88 |
+
40.684993743896484,
|
| 89 |
+
282.5484619140625,
|
| 90 |
+
47.938270568847656,
|
| 91 |
+
45.019649505615234,
|
| 92 |
+
268.04327392578125,
|
| 93 |
+
51.39228820800781,
|
| 94 |
+
41.36457443237305,
|
| 95 |
+
54.01919937133789,
|
| 96 |
+
43.2599983215332,
|
| 97 |
+
47.13923263549805,
|
| 98 |
+
41.1484489440918,
|
| 99 |
+
35.99542236328125,
|
| 100 |
+
80.46995544433594,
|
| 101 |
+
83.43251037597656,
|
| 102 |
+
48.88550567626953,
|
| 103 |
+
171.4989776611328,
|
| 104 |
+
58.917728424072266,
|
| 105 |
+
46.90486526489258,
|
| 106 |
+
56.416385650634766,
|
| 107 |
+
44.49577713012695,
|
| 108 |
+
49.42009735107422,
|
| 109 |
+
43.6354866027832,
|
| 110 |
+
55.22761535644531,
|
| 111 |
+
48.02000045776367,
|
| 112 |
+
39.44412612915039,
|
| 113 |
+
30.26905632019043,
|
| 114 |
+
492.74456787109375,
|
| 115 |
+
45.74156951904297,
|
| 116 |
+
39.223392486572266,
|
| 117 |
+
50.18822479248047,
|
| 118 |
+
43.890113830566406,
|
| 119 |
+
51.966495513916016,
|
| 120 |
+
47.814815521240234,
|
| 121 |
+
54.13961410522461,
|
| 122 |
+
39.215328216552734,
|
| 123 |
+
56.004703521728516,
|
| 124 |
+
39.528831481933594,
|
| 125 |
+
396.5307312011719,
|
| 126 |
+
83.4832992553711,
|
| 127 |
+
50.0503044128418,
|
| 128 |
+
153.1714630126953,
|
| 129 |
+
61.07219314575195,
|
| 130 |
+
52.57244873046875,
|
| 131 |
+
42.75983810424805,
|
| 132 |
+
38.4617919921875,
|
| 133 |
+
44.08700942993164,
|
| 134 |
+
53.36006546020508,
|
| 135 |
+
45.519649505615234,
|
| 136 |
+
290.8823547363281,
|
| 137 |
+
48.10774230957031,
|
| 138 |
+
53.34476852416992,
|
| 139 |
+
51.016319274902344,
|
| 140 |
+
42.9719352722168
|
| 141 |
+
],
|
| 142 |
+
"end_to_end_latency_ms_per_sample": [
|
| 143 |
+
35.17372906208038,
|
| 144 |
+
48.07086195796728,
|
| 145 |
+
46.07532289810479,
|
| 146 |
+
45.34715902991593,
|
| 147 |
+
46.541396994143724,
|
| 148 |
+
45.594630064442754,
|
| 149 |
+
45.38595397025347,
|
| 150 |
+
39.57494895439595,
|
| 151 |
+
39.812199072912335,
|
| 152 |
+
51.36631103232503,
|
| 153 |
+
42.81833197455853,
|
| 154 |
+
33.49219704978168,
|
| 155 |
+
37.3103239107877,
|
| 156 |
+
38.96563302259892,
|
| 157 |
+
40.327758993953466,
|
| 158 |
+
44.32409000582993,
|
| 159 |
+
43.25311293359846,
|
| 160 |
+
33.99710892699659,
|
| 161 |
+
49.12354901898652,
|
| 162 |
+
45.043769059702754,
|
| 163 |
+
47.554407035931945,
|
| 164 |
+
37.903546006418765,
|
| 165 |
+
32.56980108562857,
|
| 166 |
+
35.192839917726815,
|
| 167 |
+
39.34567596297711,
|
| 168 |
+
46.67255701497197,
|
| 169 |
+
49.70169102307409,
|
| 170 |
+
52.48237494379282,
|
| 171 |
+
43.415413005277514,
|
| 172 |
+
48.36420400533825,
|
| 173 |
+
48.96194895263761,
|
| 174 |
+
51.6092429170385,
|
| 175 |
+
37.311508087441325,
|
| 176 |
+
50.858690054155886,
|
| 177 |
+
43.94579504150897,
|
| 178 |
+
36.09434806276113,
|
| 179 |
+
26.66287892498076,
|
| 180 |
+
41.9749760767445,
|
| 181 |
+
34.900374012067914,
|
| 182 |
+
39.04577391222119,
|
| 183 |
+
51.60050210542977,
|
| 184 |
+
46.00996105000377,
|
| 185 |
+
53.40714694466442,
|
| 186 |
+
51.78673507180065,
|
| 187 |
+
54.96632098220289,
|
| 188 |
+
46.47863702848554,
|
| 189 |
+
47.48446703888476,
|
| 190 |
+
36.593522992916405,
|
| 191 |
+
37.32479899190366,
|
| 192 |
+
26.43477905075997,
|
| 193 |
+
39.97986298054457,
|
| 194 |
+
55.87899999227375,
|
| 195 |
+
40.821335976943374,
|
| 196 |
+
45.88699492160231,
|
| 197 |
+
40.707007981836796,
|
| 198 |
+
34.627774031832814,
|
| 199 |
+
43.4188200160861,
|
| 200 |
+
49.56604307517409,
|
| 201 |
+
39.82124803587794,
|
| 202 |
+
249.81214595027268,
|
| 203 |
+
38.2277499884367,
|
| 204 |
+
48.29735495150089,
|
| 205 |
+
47.6501100929454,
|
| 206 |
+
40.76927900314331
|
| 207 |
+
],
|
| 208 |
+
"mean_forward_latency_ms_per_sample": 78.70017620921135,
|
| 209 |
+
"median_forward_latency_ms_per_sample": 49.41412925720215,
|
| 210 |
+
"mean_end_to_end_latency_ms_per_sample": 46.30809425361804,
|
| 211 |
+
"median_end_to_end_latency_ms_per_sample": 43.68230752879754,
|
| 212 |
+
"flop_profiles": [
|
| 213 |
+
{
|
| 214 |
+
"batch_size": 1,
|
| 215 |
+
"flops_counted": 116518072320,
|
| 216 |
+
"gflops_counted_per_sample": 116.51807232,
|
| 217 |
+
"operators": {
|
| 218 |
+
"aten.convolution": 116319006720,
|
| 219 |
+
"aten.bmm": 199065600
|
| 220 |
+
}
|
| 221 |
+
},
|
| 222 |
+
{
|
| 223 |
+
"batch_size": 1,
|
| 224 |
+
"flops_counted": 145709798400,
|
| 225 |
+
"gflops_counted_per_sample": 145.7097984,
|
| 226 |
+
"operators": {
|
| 227 |
+
"aten.convolution": 145398758400,
|
| 228 |
+
"aten.bmm": 311040000
|
| 229 |
+
}
|
| 230 |
+
},
|
| 231 |
+
{
|
| 232 |
+
"batch_size": 1,
|
| 233 |
+
"flops_counted": 145709798400,
|
| 234 |
+
"gflops_counted_per_sample": 145.7097984,
|
| 235 |
+
"operators": {
|
| 236 |
+
"aten.convolution": 145398758400,
|
| 237 |
+
"aten.bmm": 311040000
|
| 238 |
+
}
|
| 239 |
+
},
|
| 240 |
+
{
|
| 241 |
+
"batch_size": 1,
|
| 242 |
+
"flops_counted": 135976458240,
|
| 243 |
+
"gflops_counted_per_sample": 135.97645824,
|
| 244 |
+
"operators": {
|
| 245 |
+
"aten.convolution": 135705507840,
|
| 246 |
+
"aten.bmm": 270950400
|
| 247 |
+
}
|
| 248 |
+
}
|
| 249 |
+
],
|
| 250 |
+
"mean_counted_gflops_per_sample": 135.97853184000002,
|
| 251 |
+
"flops_scope": "GPU FlopCounterMode around the real Ultralytics predict path; multiply-add counted as 2 FLOPs. Elementwise, postprocessing and unsupported custom ops may be missing.",
|
| 252 |
+
"latency_scope": "A800 wall-clock latency for real Ultralytics prediction including image loading/preprocess, forward and result postprocessing, excluding model load; batch processing uses the recorded batch size.",
|
| 253 |
+
"completed": true
|
| 254 |
+
}
|
releases/20260913_lr/paper_data/main_table_cost_20260912/cost_compact.pdf
ADDED
|
Binary file (43.6 kB). View file
|
|
|
releases/20260913_lr/paper_data/main_table_cost_20260912/cost_compact.png
ADDED
|
Git LFS Details
|
releases/20260913_lr/paper_data/main_table_cost_20260912/cost_compact_verified.png
ADDED
|
releases/20260913_lr/paper_data/main_table_cost_20260912/cost_summary.json
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"components": {
|
| 3 |
+
"grefcoco|instancevg|MiniCPM": {
|
| 4 |
+
"specialist_gflops": 7.502680064,
|
| 5 |
+
"router_gflops": 3.642171392,
|
| 6 |
+
"large_gflops": 12916.147709832,
|
| 7 |
+
"specialist_ms": 46.08042127620138,
|
| 8 |
+
"router_ms": 13.180577338061994,
|
| 9 |
+
"large_ms": 3667.173011115665,
|
| 10 |
+
"large_samples": 64,
|
| 11 |
+
"flops_probes": 8,
|
| 12 |
+
"specialist_samples": 64,
|
| 13 |
+
"router_samples": 64,
|
| 14 |
+
"batch_size": 1
|
| 15 |
+
},
|
| 16 |
+
"construction|yolo26x|Qwen3.8": {
|
| 17 |
+
"specialist_gflops": 135.97853184000002,
|
| 18 |
+
"router_gflops": 3.183754752,
|
| 19 |
+
"large_gflops": 96874.712551641,
|
| 20 |
+
"specialist_ms": 46.30809425361804,
|
| 21 |
+
"router_ms": 25.261267359383055,
|
| 22 |
+
"large_ms": 9822.475872695577,
|
| 23 |
+
"large_samples": 64,
|
| 24 |
+
"flops_probes": 8,
|
| 25 |
+
"specialist_samples": 64,
|
| 26 |
+
"router_samples": 64,
|
| 27 |
+
"batch_size": 1
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"selected_points": [
|
| 31 |
+
{
|
| 32 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 33 |
+
"method": "learned",
|
| 34 |
+
"nominal_validation_budget": 0.23,
|
| 35 |
+
"actual_test_call_fraction": 0.30904914499617164,
|
| 36 |
+
"test_metric": 0.6671865694921022,
|
| 37 |
+
"estimated_gflops_per_input": 4002.86925782384,
|
| 38 |
+
"estimated_time_ms_per_input": 1192.5976822525959
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"pair": "grefcoco|instancevg|MiniCPM",
|
| 42 |
+
"method": "confidence",
|
| 43 |
+
"nominal_validation_budget": 0.65,
|
| 44 |
+
"actual_test_call_fraction": 0.5209426310864078,
|
| 45 |
+
"test_metric": 0.6187788900547316,
|
| 46 |
+
"estimated_gflops_per_input": 6736.074651524562,
|
| 47 |
+
"estimated_time_ms_per_input": 1956.4671783358606
|
| 48 |
+
},
|
| 49 |
+
{
|
| 50 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 51 |
+
"method": "learned",
|
| 52 |
+
"nominal_validation_budget": 0.5,
|
| 53 |
+
"actual_test_call_fraction": 0.4876830892143808,
|
| 54 |
+
"test_metric": 0.29875800068449215,
|
| 55 |
+
"estimated_gflops_per_input": 47383.321370531434,
|
| 56 |
+
"estimated_time_ms_per_input": 4861.824738942901
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"pair": "construction|yolo26x|Qwen3.8",
|
| 60 |
+
"method": "confidence",
|
| 61 |
+
"nominal_validation_budget": 0.93,
|
| 62 |
+
"actual_test_call_fraction": 0.9221038615179761,
|
| 63 |
+
"test_metric": 0.2597490004762105,
|
| 64 |
+
"estimated_gflops_per_input": 89464.5250591521,
|
| 65 |
+
"estimated_time_ms_per_input": 9103.651026133362
|
| 66 |
+
}
|
| 67 |
+
],
|
| 68 |
+
"protocol": {
|
| 69 |
+
"method": "original main-table logit contrast; original checkpoints and validation thresholds; seed 42",
|
| 70 |
+
"large_timing": "64 measured examples; batch size one; same full task prompt; two separate warmup examples",
|
| 71 |
+
"cost": "serial component estimates, not a timed integrated service; includes active distilled encoders and routing head only for learned",
|
| 72 |
+
"flops": "multiply-add=2; supported operators; mean prefill (including visual embedding) plus average generated length times one-token decode probe",
|
| 73 |
+
"scope_limits": "mean call cost extrapolated across policies; no policy-specific input/output-length costs; specialist-feature extraction overhead is not separately measured; confidence threshold overhead not separately timed"
|
| 74 |
+
}
|
| 75 |
+
}
|