| { | |
| "validation_manifest": "frozen-validation-runtime-corrected-molmo-geometry-43a36456d23afb427333eec58f4e38c58ab9f5e1450391077580f171aac8d4b9.jsonl", | |
| "rows": 1022, | |
| "errors": 0, | |
| "subset": "molmopoint + curatedVl3 + every 20th vl2; first 20 actionable-inventory rows; first 80 curated VL-3 rows", | |
| "minicpmv": { | |
| "rows": 1022, | |
| "normalizedExact": { | |
| "count": 9, | |
| "denominator": 1022, | |
| "percent": 0.8806262230919765 | |
| }, | |
| "schemaValid": { | |
| "count": 983, | |
| "denominator": 1022, | |
| "percent": 96.1839530332681 | |
| }, | |
| "invalidUnparsable": { | |
| "count": 39, | |
| "denominator": 1022, | |
| "percent": 3.816046966731898 | |
| }, | |
| "hallucinatedActionOrTool": { | |
| "count": 0, | |
| "denominator": 1022, | |
| "percent": 0.0 | |
| }, | |
| "thinkLeakage": { | |
| "count": 0, | |
| "denominator": 1022, | |
| "percent": 0.0 | |
| }, | |
| "piiOrSecret": { | |
| "count": 0, | |
| "denominator": 1022, | |
| "percent": 0.0 | |
| }, | |
| "normalizedNextGroundedActionExact": { | |
| "count": 0, | |
| "denominator": 255, | |
| "percent": 0.0 | |
| }, | |
| "primaryNextGroundedInteractionExact": { | |
| "count": 0, | |
| "denominator": 255, | |
| "percent": 0.0 | |
| }, | |
| "point": { | |
| "denominator": 510, | |
| "meanDistance": 0.15622995573478102, | |
| "accuracyWithin05": 0.6549019607843137 | |
| }, | |
| "bbox": { | |
| "denominator": 510, | |
| "meanIou": 0.1745839738472958, | |
| "meanContainment": 0.3333333333333333 | |
| }, | |
| "actionToolNameAccuracy": { | |
| "count": 0, | |
| "denominator": 0, | |
| "percent": null, | |
| "reason": "no unsealed WebChain action validation exists" | |
| }, | |
| "normalizedArgumentExact": { | |
| "count": 0, | |
| "denominator": 255, | |
| "percent": 0.0 | |
| }, | |
| "terminalOutcome": { | |
| "count": 0, | |
| "denominator": 0, | |
| "percent": null, | |
| "reason": "no terminal validation task in frozen VL suites" | |
| }, | |
| "by_task": { | |
| "WBVL_AUX_INTENT_TO_POINT_BOX_V1": { | |
| "rows": 255, | |
| "schemaValid": 100.0, | |
| "normalizedExact": 0.0, | |
| "point": { | |
| "denominator": 255, | |
| "meanDistance": 0.19030809272697904, | |
| "accuracyWithin05": 0.7372549019607844 | |
| }, | |
| "bbox": { | |
| "denominator": 255, | |
| "meanIou": 0.18179181196933797, | |
| "meanContainment": 0.37254901960784315 | |
| } | |
| }, | |
| "WBVL_AUX_POINT_BOX_TO_ELEMENT_V1": { | |
| "rows": 255, | |
| "schemaValid": 100.0, | |
| "normalizedExact": 0.39215686274509803, | |
| "point": { | |
| "denominator": 0, | |
| "meanDistance": null, | |
| "accuracyWithin05": null | |
| }, | |
| "bbox": { | |
| "denominator": 0, | |
| "meanIou": null, | |
| "meanContainment": null | |
| } | |
| }, | |
| "WBVL_AUX_SCREENSHOT_TO_ACTIONABLE_INVENTORY_V1": { | |
| "rows": 20, | |
| "schemaValid": 25.0, | |
| "normalizedExact": 0.0, | |
| "point": { | |
| "denominator": 0, | |
| "meanDistance": null, | |
| "accuracyWithin05": null | |
| }, | |
| "bbox": { | |
| "denominator": 0, | |
| "meanIou": null, | |
| "meanContainment": null | |
| } | |
| }, | |
| "WBVL_AUX_USER_INTENT_TO_GROUNDED_INTERACTION_V1": { | |
| "rows": 255, | |
| "schemaValid": 100.0, | |
| "normalizedExact": 0.0, | |
| "point": { | |
| "denominator": 255, | |
| "meanDistance": 0.12215181874258325, | |
| "accuracyWithin05": 0.5725490196078431 | |
| }, | |
| "bbox": { | |
| "denominator": 255, | |
| "meanIou": 0.1673761357252536, | |
| "meanContainment": 0.29411764705882354 | |
| } | |
| }, | |
| "WBVL_SIX_SECTION_OBSERVATION_V2": { | |
| "rows": 157, | |
| "schemaValid": 98.72611464968153, | |
| "normalizedExact": 5.095541401273885, | |
| "point": { | |
| "denominator": 0, | |
| "meanDistance": null, | |
| "accuracyWithin05": null | |
| }, | |
| "bbox": { | |
| "denominator": 0, | |
| "meanIou": null, | |
| "meanContainment": null | |
| } | |
| }, | |
| "WBVL_SIX_SECTION_OBSERVATION_V3": { | |
| "rows": 80, | |
| "schemaValid": 72.5, | |
| "normalizedExact": 0.0, | |
| "point": { | |
| "denominator": 0, | |
| "meanDistance": null, | |
| "accuracyWithin05": null | |
| }, | |
| "bbox": { | |
| "denominator": 0, | |
| "meanIou": null, | |
| "meanContainment": null | |
| } | |
| } | |
| } | |
| }, | |
| "lfm25_vl_450m_step3750_same_rows": { | |
| "rows": 1022, | |
| "normalizedExact": { | |
| "count": 0, | |
| "denominator": 1022, | |
| "percent": 0.0 | |
| }, | |
| "schemaValid": { | |
| "count": 947, | |
| "denominator": 1022, | |
| "percent": 92.66144814090019 | |
| }, | |
| "invalidUnparsable": { | |
| "count": 73, | |
| "denominator": 1022, | |
| "percent": 7.142857142857143 | |
| }, | |
| "hallucinatedActionOrTool": { | |
| "count": 0, | |
| "denominator": 1022, | |
| "percent": 0.0 | |
| }, | |
| "thinkLeakage": { | |
| "count": 0, | |
| "denominator": 1022, | |
| "percent": 0.0 | |
| }, | |
| "piiOrSecret": { | |
| "count": 0, | |
| "denominator": 1022, | |
| "percent": 0.0 | |
| }, | |
| "normalizedNextGroundedActionExact": { | |
| "count": 0, | |
| "denominator": 255, | |
| "percent": 0.0 | |
| }, | |
| "primaryNextGroundedInteractionExact": { | |
| "count": 0, | |
| "denominator": 255, | |
| "percent": 0.0 | |
| }, | |
| "point": { | |
| "denominator": 508, | |
| "meanDistance": 0.16054613245167407, | |
| "accuracyWithin05": 0.45866141732283466 | |
| }, | |
| "bbox": { | |
| "denominator": 510, | |
| "meanIou": 0.04957207453569697, | |
| "meanContainment": 0.10433070866141732 | |
| }, | |
| "actionToolNameAccuracy": { | |
| "count": 0, | |
| "denominator": 0, | |
| "percent": null, | |
| "reason": "no unsealed WebChain action validation exists" | |
| }, | |
| "normalizedArgumentExact": { | |
| "count": 0, | |
| "denominator": 255, | |
| "percent": 0.0 | |
| }, | |
| "terminalOutcome": { | |
| "count": 0, | |
| "denominator": 0, | |
| "percent": null, | |
| "reason": "no terminal validation task in frozen VL suites" | |
| }, | |
| "by_task": { | |
| "WBVL_AUX_INTENT_TO_POINT_BOX_V1": { | |
| "rows": 255, | |
| "schemaValid": 99.2156862745098, | |
| "normalizedExact": 0.0, | |
| "point": { | |
| "denominator": 253, | |
| "meanDistance": 0.13226767855607902, | |
| "accuracyWithin05": 0.6205533596837944 | |
| }, | |
| "bbox": { | |
| "denominator": 255, | |
| "meanIou": 0.056399046894241606, | |
| "meanContainment": 0.11857707509881422 | |
| } | |
| }, | |
| "WBVL_AUX_POINT_BOX_TO_ELEMENT_V1": { | |
| "rows": 255, | |
| "schemaValid": 99.2156862745098, | |
| "normalizedExact": 0.0, | |
| "point": { | |
| "denominator": 0, | |
| "meanDistance": null, | |
| "accuracyWithin05": null | |
| }, | |
| "bbox": { | |
| "denominator": 0, | |
| "meanIou": null, | |
| "meanContainment": null | |
| } | |
| }, | |
| "WBVL_AUX_SCREENSHOT_TO_ACTIONABLE_INVENTORY_V1": { | |
| "rows": 20, | |
| "schemaValid": 10.0, | |
| "normalizedExact": 0.0, | |
| "point": { | |
| "denominator": 0, | |
| "meanDistance": null, | |
| "accuracyWithin05": null | |
| }, | |
| "bbox": { | |
| "denominator": 0, | |
| "meanIou": null, | |
| "meanContainment": null | |
| } | |
| }, | |
| "WBVL_AUX_USER_INTENT_TO_GROUNDED_INTERACTION_V1": { | |
| "rows": 255, | |
| "schemaValid": 100.0, | |
| "normalizedExact": 0.0, | |
| "point": { | |
| "denominator": 255, | |
| "meanDistance": 0.18860279455200923, | |
| "accuracyWithin05": 0.2980392156862745 | |
| }, | |
| "bbox": { | |
| "denominator": 255, | |
| "meanIou": 0.042745102177152404, | |
| "meanContainment": 0.09019607843137255 | |
| } | |
| }, | |
| "WBVL_SIX_SECTION_OBSERVATION_V2": { | |
| "rows": 157, | |
| "schemaValid": 97.45222929936305, | |
| "normalizedExact": 0.0, | |
| "point": { | |
| "denominator": 0, | |
| "meanDistance": null, | |
| "accuracyWithin05": null | |
| }, | |
| "bbox": { | |
| "denominator": 0, | |
| "meanIou": null, | |
| "meanContainment": null | |
| } | |
| }, | |
| "WBVL_SIX_SECTION_OBSERVATION_V3": { | |
| "rows": 80, | |
| "schemaValid": 38.75, | |
| "normalizedExact": 0.0, | |
| "point": { | |
| "denominator": 0, | |
| "meanDistance": null, | |
| "accuracyWithin05": null | |
| }, | |
| "bbox": { | |
| "denominator": 0, | |
| "meanIou": null, | |
| "meanContainment": null | |
| } | |
| } | |
| } | |
| }, | |
| "lfm_rows_matched": 1022, | |
| "previous_same_rows": { | |
| "rows": 1022, | |
| "normalizedExact": { | |
| "count": 7, | |
| "denominator": 1022, | |
| "percent": 0.684931506849315 | |
| }, | |
| "schemaValid": { | |
| "count": 979, | |
| "denominator": 1022, | |
| "percent": 95.79256360078278 | |
| }, | |
| "invalidUnparsable": { | |
| "count": 43, | |
| "denominator": 1022, | |
| "percent": 4.2074363992172215 | |
| }, | |
| "hallucinatedActionOrTool": { | |
| "count": 0, | |
| "denominator": 1022, | |
| "percent": 0.0 | |
| }, | |
| "thinkLeakage": { | |
| "count": 0, | |
| "denominator": 1022, | |
| "percent": 0.0 | |
| }, | |
| "piiOrSecret": { | |
| "count": 0, | |
| "denominator": 1022, | |
| "percent": 0.0 | |
| }, | |
| "normalizedNextGroundedActionExact": { | |
| "count": 0, | |
| "denominator": 255, | |
| "percent": 0.0 | |
| }, | |
| "primaryNextGroundedInteractionExact": { | |
| "count": 0, | |
| "denominator": 255, | |
| "percent": 0.0 | |
| }, | |
| "point": { | |
| "denominator": 510, | |
| "meanDistance": 0.193548109525868, | |
| "accuracyWithin05": 0.5882352941176471 | |
| }, | |
| "bbox": { | |
| "denominator": 510, | |
| "meanIou": 0.11283005443699187, | |
| "meanContainment": 0.23333333333333334 | |
| }, | |
| "actionToolNameAccuracy": { | |
| "count": 0, | |
| "denominator": 0, | |
| "percent": null, | |
| "reason": "no unsealed WebChain action validation exists" | |
| }, | |
| "normalizedArgumentExact": { | |
| "count": 0, | |
| "denominator": 255, | |
| "percent": 0.0 | |
| }, | |
| "terminalOutcome": { | |
| "count": 0, | |
| "denominator": 0, | |
| "percent": null, | |
| "reason": "no terminal validation task in frozen VL suites" | |
| }, | |
| "by_task": { | |
| "WBVL_AUX_INTENT_TO_POINT_BOX_V1": { | |
| "rows": 255, | |
| "schemaValid": 100.0, | |
| "normalizedExact": 0.0, | |
| "point": { | |
| "denominator": 255, | |
| "meanDistance": 0.24760412207618865, | |
| "accuracyWithin05": 0.6666666666666666 | |
| }, | |
| "bbox": { | |
| "denominator": 255, | |
| "meanIou": 0.09999878331361452, | |
| "meanContainment": 0.22745098039215686 | |
| } | |
| }, | |
| "WBVL_AUX_POINT_BOX_TO_ELEMENT_V1": { | |
| "rows": 255, | |
| "schemaValid": 100.0, | |
| "normalizedExact": 0.39215686274509803, | |
| "point": { | |
| "denominator": 0, | |
| "meanDistance": null, | |
| "accuracyWithin05": null | |
| }, | |
| "bbox": { | |
| "denominator": 0, | |
| "meanIou": null, | |
| "meanContainment": null | |
| } | |
| }, | |
| "WBVL_AUX_SCREENSHOT_TO_ACTIONABLE_INVENTORY_V1": { | |
| "rows": 20, | |
| "schemaValid": 30.0, | |
| "normalizedExact": 0.0, | |
| "point": { | |
| "denominator": 0, | |
| "meanDistance": null, | |
| "accuracyWithin05": null | |
| }, | |
| "bbox": { | |
| "denominator": 0, | |
| "meanIou": null, | |
| "meanContainment": null | |
| } | |
| }, | |
| "WBVL_AUX_USER_INTENT_TO_GROUNDED_INTERACTION_V1": { | |
| "rows": 255, | |
| "schemaValid": 100.0, | |
| "normalizedExact": 0.0, | |
| "point": { | |
| "denominator": 255, | |
| "meanDistance": 0.13949209697554749, | |
| "accuracyWithin05": 0.5098039215686274 | |
| }, | |
| "bbox": { | |
| "denominator": 255, | |
| "meanIou": 0.1256613255603694, | |
| "meanContainment": 0.23921568627450981 | |
| } | |
| }, | |
| "WBVL_SIX_SECTION_OBSERVATION_V2": { | |
| "rows": 157, | |
| "schemaValid": 99.36305732484077, | |
| "normalizedExact": 3.821656050955414, | |
| "point": { | |
| "denominator": 0, | |
| "meanDistance": null, | |
| "accuracyWithin05": null | |
| }, | |
| "bbox": { | |
| "denominator": 0, | |
| "meanIou": null, | |
| "meanContainment": null | |
| } | |
| }, | |
| "WBVL_SIX_SECTION_OBSERVATION_V3": { | |
| "rows": 80, | |
| "schemaValid": 65.0, | |
| "normalizedExact": 0.0, | |
| "point": { | |
| "denominator": 0, | |
| "meanDistance": null, | |
| "accuracyWithin05": null | |
| }, | |
| "bbox": { | |
| "denominator": 0, | |
| "meanIou": null, | |
| "meanContainment": null | |
| } | |
| } | |
| } | |
| }, | |
| "previous_raw": "raw.jsonl", | |
| "seconds_per_row_pytorch_bf16": { | |
| "WBVL_AUX_INTENT_TO_POINT_BOX_V1": 1.94, | |
| "WBVL_AUX_POINT_BOX_TO_ELEMENT_V1": 0.704, | |
| "WBVL_AUX_SCREENSHOT_TO_ACTIONABLE_INVENTORY_V1": 52.118, | |
| "WBVL_AUX_USER_INTENT_TO_GROUNDED_INTERACTION_V1": 2.332, | |
| "WBVL_SIX_SECTION_OBSERVATION_V2": 7.128, | |
| "WBVL_SIX_SECTION_OBSERVATION_V3": 16.07 | |
| } | |
| } |