Spaces:
Running on Zero
Running on Zero
Replace gallery with 22 original cases and support point-in-mask tasks
Browse filesSync the local groundingpi-personal-demo case update. Replace gallery metadata and assets with the 22 original_images cases across 9 tasks, preserving original image bytes and dimensions. Add point-in-mask prompt and protocol support and lock the updated UI. Verified locally: 28 relevant tests and UI consistency checks passed.
- .gitattributes +20 -0
- README.md +4 -2
- app.py +2 -2
- cases/dense/04_dense_round2_gam_dense200_66.jpg +3 -0
- cases/dense/05_dense_round2_gam_dense200_77.jpg +3 -0
- cases/dense/06_dense_round2_gam_dense200_160.jpg +3 -0
- cases/dense_point/12_dense_point_demo_cases_demo_dense_point_9.jpg +3 -0
- cases/dense_point/13_dense_point_demo_cases_demo_dense_point_10.jpg +3 -0
- cases/dense_point_in_mask/14_dense_point_in_mask_round2_gam_rex_point_visdrone_190.jpg +3 -0
- cases/grounding/01_grounding_demo_cases_demo_dense_5.jpg +3 -0
- cases/grounding/02_grounding_demo_cases_demo_grounding_0.jpg +0 -0
- cases/grounding/03_grounding_demo_cases_demo_grounding_1.jpg +0 -0
- cases/gui/15_gui_round5_candidates_gam_screenspot_pro_694.png +3 -0
- cases/gui/16_gui_round5_candidates_gam_screenspot_pro_1164.png +3 -0
- cases/ocr/17_ocr_round3_ocr_gam_hiertext_859.jpg +3 -0
- cases/ocr/18_ocr_round5_candidates_gam_sroie_38.jpg +3 -0
- cases/ocr/19_ocr_round3_ocr_gam_totaltext_54.jpg +3 -0
- cases/referring/07_referring_round5_candidates_gam_humanref_1625.jpg +3 -0
- cases/referring/08_referring_round5_candidates_gam_humanref_1856.jpg +3 -0
- cases/referring/09_referring_round5_candidates_gam_humanref_2178.jpg +3 -0
- cases/referring_point_in_mask/10_referring_point_in_mask_round4_candidates_gam_rex_point_humanref_1164.jpg +3 -0
- cases/referring_point_in_mask/11_referring_point_in_mask_round5_candidates_gam_rex_point_refcocog_test_8433.jpg +3 -0
- cases/visual_prompt/20_visual_prompt_demo_cases_demo_visual_prompt_15.jpg +3 -0
- cases/visual_prompt/21_visual_prompt_round5_candidates_gam_visual_dense200_142.jpg +3 -0
- cases/visual_prompt/22_visual_prompt_round5_candidates_gam_visual_dense200_187.jpg +3 -0
- data/cases.json +0 -0
- docs/ui_case_pool_22_lock.json +43 -0
- grounding_demo/console.py +13 -11
- grounding_demo/protocol.py +1 -1
- scripts/build_bundle.py +5 -2
- scripts/preflight_model.py +1 -1
- scripts/verify_ui.py +6 -3
- tests/test_bundle.py +13 -1
- tests/test_console.py +12 -1
- tests/test_protocol.py +8 -0
.gitattributes
CHANGED
|
@@ -54,3 +54,23 @@ cases/referring/referring-03-humanref_nocot_lf2_00011458.jpg filter=lfs diff=lfs
|
|
| 54 |
cases/visual_prompt/visual_prompt-01-sensenova_sa1b_visual_1718306.jpg filter=lfs diff=lfs merge=lfs -text
|
| 55 |
cases/visual_prompt/visual_prompt-02-sensenova_sa1b_visual_2475968.jpg filter=lfs diff=lfs merge=lfs -text
|
| 56 |
cases/visual_prompt/visual_prompt-03-sensenova_sa1b_visual_1573415.jpg filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
cases/visual_prompt/visual_prompt-01-sensenova_sa1b_visual_1718306.jpg filter=lfs diff=lfs merge=lfs -text
|
| 55 |
cases/visual_prompt/visual_prompt-02-sensenova_sa1b_visual_2475968.jpg filter=lfs diff=lfs merge=lfs -text
|
| 56 |
cases/visual_prompt/visual_prompt-03-sensenova_sa1b_visual_1573415.jpg filter=lfs diff=lfs merge=lfs -text
|
| 57 |
+
cases/dense_point_in_mask/14_dense_point_in_mask_round2_gam_rex_point_visdrone_190.jpg filter=lfs diff=lfs merge=lfs -text
|
| 58 |
+
cases/dense_point/12_dense_point_demo_cases_demo_dense_point_9.jpg filter=lfs diff=lfs merge=lfs -text
|
| 59 |
+
cases/dense_point/13_dense_point_demo_cases_demo_dense_point_10.jpg filter=lfs diff=lfs merge=lfs -text
|
| 60 |
+
cases/dense/04_dense_round2_gam_dense200_66.jpg filter=lfs diff=lfs merge=lfs -text
|
| 61 |
+
cases/dense/05_dense_round2_gam_dense200_77.jpg filter=lfs diff=lfs merge=lfs -text
|
| 62 |
+
cases/dense/06_dense_round2_gam_dense200_160.jpg filter=lfs diff=lfs merge=lfs -text
|
| 63 |
+
cases/grounding/01_grounding_demo_cases_demo_dense_5.jpg filter=lfs diff=lfs merge=lfs -text
|
| 64 |
+
cases/gui/15_gui_round5_candidates_gam_screenspot_pro_694.png filter=lfs diff=lfs merge=lfs -text
|
| 65 |
+
cases/gui/16_gui_round5_candidates_gam_screenspot_pro_1164.png filter=lfs diff=lfs merge=lfs -text
|
| 66 |
+
cases/ocr/17_ocr_round3_ocr_gam_hiertext_859.jpg filter=lfs diff=lfs merge=lfs -text
|
| 67 |
+
cases/ocr/18_ocr_round5_candidates_gam_sroie_38.jpg filter=lfs diff=lfs merge=lfs -text
|
| 68 |
+
cases/ocr/19_ocr_round3_ocr_gam_totaltext_54.jpg filter=lfs diff=lfs merge=lfs -text
|
| 69 |
+
cases/referring_point_in_mask/10_referring_point_in_mask_round4_candidates_gam_rex_point_humanref_1164.jpg filter=lfs diff=lfs merge=lfs -text
|
| 70 |
+
cases/referring_point_in_mask/11_referring_point_in_mask_round5_candidates_gam_rex_point_refcocog_test_8433.jpg filter=lfs diff=lfs merge=lfs -text
|
| 71 |
+
cases/referring/07_referring_round5_candidates_gam_humanref_1625.jpg filter=lfs diff=lfs merge=lfs -text
|
| 72 |
+
cases/referring/08_referring_round5_candidates_gam_humanref_1856.jpg filter=lfs diff=lfs merge=lfs -text
|
| 73 |
+
cases/referring/09_referring_round5_candidates_gam_humanref_2178.jpg filter=lfs diff=lfs merge=lfs -text
|
| 74 |
+
cases/visual_prompt/20_visual_prompt_demo_cases_demo_visual_prompt_15.jpg filter=lfs diff=lfs merge=lfs -text
|
| 75 |
+
cases/visual_prompt/21_visual_prompt_round5_candidates_gam_visual_dense200_142.jpg filter=lfs diff=lfs merge=lfs -text
|
| 76 |
+
cases/visual_prompt/22_visual_prompt_round5_candidates_gam_visual_dense200_187.jpg filter=lfs diff=lfs merge=lfs -text
|
README.md
CHANGED
|
@@ -34,10 +34,10 @@ see [license status](LICENSE_PENDING.md).
|
|
| 34 |
| Dense | Localize instances in crowded or small-object scenes |
|
| 35 |
| Referring | Describe a particular object or person |
|
| 36 |
| Dense Point | Locate multiple targets as image-relative points |
|
| 37 |
-
| Referring Point |
|
|
|
|
| 38 |
| Visual Prompt | Supply an input region as a visual condition |
|
| 39 |
| OCR | Read text with its associated location |
|
| 40 |
-
| Layout | Localize document regions |
|
| 41 |
| GUI | Point to an interface target; no UI action is executed |
|
| 42 |
|
| 43 |
1. Select a task and a curated example, or upload your own image.
|
|
@@ -51,6 +51,8 @@ view switch. Large regions use unfilled outlines; labels avoid obscuring small
|
|
| 51 |
targets. The renderer does not delete, merge or relocate predictions for appearance.
|
| 52 |
Visual Prompt's marked region is an **input condition**, not a model prediction.
|
| 53 |
|
|
|
|
|
|
|
| 54 |
Examples are qualitative scenarios selected from training pools, not a held-out
|
| 55 |
benchmark. An operational demo is not evidence of a reported accuracy or speedup.
|
| 56 |
|
|
|
|
| 34 |
| Dense | Localize instances in crowded or small-object scenes |
|
| 35 |
| Referring | Describe a particular object or person |
|
| 36 |
| Dense Point | Locate multiple targets as image-relative points |
|
| 37 |
+
| Referring Point in Mask | Predict a point inside a target described by a referring expression |
|
| 38 |
+
| Dense Point in Mask | Predict an interior point for each matching instance |
|
| 39 |
| Visual Prompt | Supply an input region as a visual condition |
|
| 40 |
| OCR | Read text with its associated location |
|
|
|
|
| 41 |
| GUI | Point to an interface target; no UI action is executed |
|
| 42 |
|
| 43 |
1. Select a task and a curated example, or upload your own image.
|
|
|
|
| 51 |
targets. The renderer does not delete, merge or relocate predictions for appearance.
|
| 52 |
Visual Prompt's marked region is an **input condition**, not a model prediction.
|
| 53 |
|
| 54 |
+
The gallery contains 22 original images from the selected candidate pool, with
|
| 55 |
+
their corresponding task prompts. Original image formats and dimensions are preserved.
|
| 56 |
Examples are qualitative scenarios selected from training pools, not a held-out
|
| 57 |
benchmark. An operational demo is not evidence of a reported accuracy or speedup.
|
| 58 |
|
app.py
CHANGED
|
@@ -21,10 +21,10 @@ ROUTE_META = {
|
|
| 21 |
"dense": ("Dense", "Dense instance localization"),
|
| 22 |
"referring": ("Referring", "Language-guided referring"),
|
| 23 |
"dense_point": ("Dense Point", "Dense point prediction"),
|
| 24 |
-
"
|
|
|
|
| 25 |
"visual_prompt": ("Visual Prompt", "Exemplar-guided localization"),
|
| 26 |
"ocr": ("OCR", "Text recognition and localization"),
|
| 27 |
-
"layout": ("Layout", "Document structure analysis"),
|
| 28 |
"gui": ("GUI", "Interface element grounding"),
|
| 29 |
}
|
| 30 |
CASES = load_cases()
|
|
|
|
| 21 |
"dense": ("Dense", "Dense instance localization"),
|
| 22 |
"referring": ("Referring", "Language-guided referring"),
|
| 23 |
"dense_point": ("Dense Point", "Dense point prediction"),
|
| 24 |
+
"referring_point_in_mask": ("Referring Point in Mask", "Point inside a referred target"),
|
| 25 |
+
"dense_point_in_mask": ("Dense Point in Mask", "Point inside multiple targets"),
|
| 26 |
"visual_prompt": ("Visual Prompt", "Exemplar-guided localization"),
|
| 27 |
"ocr": ("OCR", "Text recognition and localization"),
|
|
|
|
| 28 |
"gui": ("GUI", "Interface element grounding"),
|
| 29 |
}
|
| 30 |
CASES = load_cases()
|
cases/dense/04_dense_round2_gam_dense200_66.jpg
ADDED
|
Git LFS Details
|
cases/dense/05_dense_round2_gam_dense200_77.jpg
ADDED
|
Git LFS Details
|
cases/dense/06_dense_round2_gam_dense200_160.jpg
ADDED
|
Git LFS Details
|
cases/dense_point/12_dense_point_demo_cases_demo_dense_point_9.jpg
ADDED
|
Git LFS Details
|
cases/dense_point/13_dense_point_demo_cases_demo_dense_point_10.jpg
ADDED
|
Git LFS Details
|
cases/dense_point_in_mask/14_dense_point_in_mask_round2_gam_rex_point_visdrone_190.jpg
ADDED
|
Git LFS Details
|
cases/grounding/01_grounding_demo_cases_demo_dense_5.jpg
ADDED
|
Git LFS Details
|
cases/grounding/02_grounding_demo_cases_demo_grounding_0.jpg
ADDED
|
cases/grounding/03_grounding_demo_cases_demo_grounding_1.jpg
ADDED
|
cases/gui/15_gui_round5_candidates_gam_screenspot_pro_694.png
ADDED
|
Git LFS Details
|
cases/gui/16_gui_round5_candidates_gam_screenspot_pro_1164.png
ADDED
|
Git LFS Details
|
cases/ocr/17_ocr_round3_ocr_gam_hiertext_859.jpg
ADDED
|
Git LFS Details
|
cases/ocr/18_ocr_round5_candidates_gam_sroie_38.jpg
ADDED
|
Git LFS Details
|
cases/ocr/19_ocr_round3_ocr_gam_totaltext_54.jpg
ADDED
|
Git LFS Details
|
cases/referring/07_referring_round5_candidates_gam_humanref_1625.jpg
ADDED
|
Git LFS Details
|
cases/referring/08_referring_round5_candidates_gam_humanref_1856.jpg
ADDED
|
Git LFS Details
|
cases/referring/09_referring_round5_candidates_gam_humanref_2178.jpg
ADDED
|
Git LFS Details
|
cases/referring_point_in_mask/10_referring_point_in_mask_round4_candidates_gam_rex_point_humanref_1164.jpg
ADDED
|
Git LFS Details
|
cases/referring_point_in_mask/11_referring_point_in_mask_round5_candidates_gam_rex_point_refcocog_test_8433.jpg
ADDED
|
Git LFS Details
|
cases/visual_prompt/20_visual_prompt_demo_cases_demo_visual_prompt_15.jpg
ADDED
|
Git LFS Details
|
cases/visual_prompt/21_visual_prompt_round5_candidates_gam_visual_dense200_142.jpg
ADDED
|
Git LFS Details
|
cases/visual_prompt/22_visual_prompt_round5_candidates_gam_visual_dense200_187.jpg
ADDED
|
Git LFS Details
|
data/cases.json
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
docs/ui_case_pool_22_lock.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"files": {
|
| 3 |
+
"app.py": "2c98f90cb32f908dd94012d7ee2261c3a5097c33a65b06150fff68b60aa4f417",
|
| 4 |
+
"assets/Manrope-OFL.txt": "e01b637272e0cbdfb240184dd98ea5cc671556d9894dae2668d92ab2c906787c",
|
| 5 |
+
"assets/Manrope-variable.ttf": "d0639be45d0af36e798172419d7bd173c4bd4f29e2b76cbb69db1d11bf8b0a40",
|
| 6 |
+
"assets/SOURCES.md": "3cceb52136b55f86840e9585e3c37faed1adc762452b0fea50aecba696dac7fa",
|
| 7 |
+
"assets/console.js": "3af36a3aeafb72e85018b1c663f624b92877bad0152ee9e264a1c440d689f60e",
|
| 8 |
+
"assets/frosted.css": "393bf5f1c04a49a6e9b17e855e573646489b4add85bc7973eb6bb277686b5c80",
|
| 9 |
+
"assets/original.css": "7a244c7f65e40797850dcde7d6b6505b1b5de60aa24b129f44b33ae52291fd2e",
|
| 10 |
+
"assets/paper.css": "61fc60bf8c379a0894fbfe06574964ec2ed256791a23860fa4dbb872eb2869f9",
|
| 11 |
+
"assets/studio.css": "dcc22a94afbcff8079c5bcaeda2849f75e6ad7b204af9a49f2c3d954c3df46cd",
|
| 12 |
+
"assets/xpeng-logo.png": "ff0db78ddf8f9a122eec74e8eb79581f8e5dbc46803dddeb780a06f5ae295750",
|
| 13 |
+
"cases/dense/04_dense_round2_gam_dense200_66.jpg": "c12e0a9cfce7f78ec2f95cff4ae7203da8e8db8c382b9818ed0e8463aaa84c01",
|
| 14 |
+
"cases/dense/05_dense_round2_gam_dense200_77.jpg": "8a1b5031cd6d3f066cc84c99bd1e272994ebb0989f77dd3e978d2acc888cc6b6",
|
| 15 |
+
"cases/dense/06_dense_round2_gam_dense200_160.jpg": "f5f2c69d996a3d9bd5cb3369ca2de1d9f1d4188e1d3e35b4f2a824d3b3c9d9c9",
|
| 16 |
+
"cases/dense_point/12_dense_point_demo_cases_demo_dense_point_9.jpg": "917975c9b31791290a2a660c91af8ae0653cfe258fa0c53539bcb868867aece2",
|
| 17 |
+
"cases/dense_point/13_dense_point_demo_cases_demo_dense_point_10.jpg": "0b9a2281fec580c02b9292d8b3f96b675a8659d721f4dd8a1ae16a1eadbaf4e9",
|
| 18 |
+
"cases/dense_point_in_mask/14_dense_point_in_mask_round2_gam_rex_point_visdrone_190.jpg": "cfa6aa7e7c9d9d5dcd1ae79f7195106b936e7a09b07851909c294b84c521674a",
|
| 19 |
+
"cases/grounding/01_grounding_demo_cases_demo_dense_5.jpg": "eb9331146b66a1af1e8c832a22043c6f1b45d5d3bb7231816f975ced2cd88e39",
|
| 20 |
+
"cases/grounding/02_grounding_demo_cases_demo_grounding_0.jpg": "2e3946eafcbdc2d2ac9e0b62343595e0b51e71b38eb15c3b0669fab46c78e9f6",
|
| 21 |
+
"cases/grounding/03_grounding_demo_cases_demo_grounding_1.jpg": "c6abef94a28290ca94e8dd043ba58e326cfcb22088b75f32274acc07af15950c",
|
| 22 |
+
"cases/gui/15_gui_round5_candidates_gam_screenspot_pro_694.png": "38f303a6489b815474d9a502191852a9a2d3438818e4e36bd6a8760a604b064d",
|
| 23 |
+
"cases/gui/16_gui_round5_candidates_gam_screenspot_pro_1164.png": "95115faa9508f087794480b2b87f1da2acdec744c32769ea0fe312ec9f14e608",
|
| 24 |
+
"cases/ocr/17_ocr_round3_ocr_gam_hiertext_859.jpg": "2c32d77500a463f67826423a826b7b22cd0b84b8465aa99d443669172ea063d2",
|
| 25 |
+
"cases/ocr/18_ocr_round5_candidates_gam_sroie_38.jpg": "2066e659870274b0853dce4239217730fbc913d33ff2204ff1e871ccdea829ed",
|
| 26 |
+
"cases/ocr/19_ocr_round3_ocr_gam_totaltext_54.jpg": "e15821aea42dd39bc60c6865114bb9593a4676a0b824a126282767d62b950300",
|
| 27 |
+
"cases/referring/07_referring_round5_candidates_gam_humanref_1625.jpg": "52fc1d2e6337603e3059588b3eb7d14c8e9e042bee34f711bea6d8e540b01f40",
|
| 28 |
+
"cases/referring/08_referring_round5_candidates_gam_humanref_1856.jpg": "9d1550c477a7d4de6dd4d2e61d505bf04e3a4cdcdae72d7ba8879f6bd2948d1e",
|
| 29 |
+
"cases/referring/09_referring_round5_candidates_gam_humanref_2178.jpg": "213da3e5da69744c0629b6bf1a516e476a27bb3dbce5452559dfefdb30b6b9af",
|
| 30 |
+
"cases/referring_point_in_mask/10_referring_point_in_mask_round4_candidates_gam_rex_point_humanref_1164.jpg": "b5b19175f8c7e358189c122cd378ef601898363e3f3717c27fb83bf0378b5f02",
|
| 31 |
+
"cases/referring_point_in_mask/11_referring_point_in_mask_round5_candidates_gam_rex_point_refcocog_test_8433.jpg": "51ee607083e418776b73f3943bd0b72bba34c7863781adad883a816aa7c3393a",
|
| 32 |
+
"cases/visual_prompt/20_visual_prompt_demo_cases_demo_visual_prompt_15.jpg": "57104d07c82cfb97a72075ef479f7d8d1c00db82386aa5c38da0ef163a82dc2f",
|
| 33 |
+
"cases/visual_prompt/21_visual_prompt_round5_candidates_gam_visual_dense200_142.jpg": "34101e5a696ca089b3d6026fbafda21ba8a144bb2b4d4f36fd309273ed48e286",
|
| 34 |
+
"cases/visual_prompt/22_visual_prompt_round5_candidates_gam_visual_dense200_187.jpg": "3daa8e2776a2dd0ab17abebd669d95728f64bfbd063986f14becfb3f1966463a",
|
| 35 |
+
"grounding_demo/catalog.py": "75e543fed7f0ca018fb41c4b528efc2901aa6a0f05232959297d6d40798f676f",
|
| 36 |
+
"grounding_demo/console.py": "5754564abe58099c9406de639663d9d1416b96bc65d7a89b24f402ec41978fb8",
|
| 37 |
+
"grounding_demo/protocol.py": "cf043c9f026a1dcd63d55c5393f0d0921cd15eae4a0cc9a42e2834fab23486ec",
|
| 38 |
+
"grounding_demo/render.py": "ba6201d2eda2f3bf7fb0988e35f76303a4c43455d18cc60c4db547d87155d73d"
|
| 39 |
+
},
|
| 40 |
+
"case_contract_sha256": "006774de24ef12d7cd8a05f73e27ec2d27278ba63fbe000db368541f946dec93",
|
| 41 |
+
"cases": 22,
|
| 42 |
+
"routes": 9
|
| 43 |
+
}
|
grounding_demo/console.py
CHANGED
|
@@ -14,10 +14,12 @@ PREFIXES = {
|
|
| 14 |
"referring": "Locate the target referred to by the following description: ",
|
| 15 |
"dense_point": "Point to: ",
|
| 16 |
"refer_point": "Point to the target referred to by the following description: ",
|
|
|
|
|
|
|
| 17 |
"layout": "Detect all document layout elements that match the following categories: ",
|
| 18 |
"gui": "Point to the UI element to click for the following instruction: ",
|
| 19 |
}
|
| 20 |
-
CATEGORY_ROUTES = {"grounding", "dense", "layout"}
|
| 21 |
|
| 22 |
# Presentation only: never alter the released model prompt or expected answer.
|
| 23 |
TASK_HELP = {
|
|
@@ -45,11 +47,11 @@ TASK_HELP = {
|
|
| 45 |
"Describe the targets to locate. Points indicate positions, not object boundaries.",
|
| 46 |
"All visible people"
|
| 47 |
],
|
| 48 |
-
"
|
| 49 |
-
"
|
| 50 |
-
"
|
| 51 |
-
"Describe
|
| 52 |
-
"The
|
| 53 |
],
|
| 54 |
"visual_prompt": [
|
| 55 |
"Grounding by Visual Example",
|
|
@@ -63,11 +65,11 @@ TASK_HELP = {
|
|
| 63 |
"Use an OCR instruction. Select a transcription to inspect the original pixels.",
|
| 64 |
"Read and locate the text in the image"
|
| 65 |
],
|
| 66 |
-
"
|
| 67 |
-
"
|
| 68 |
-
"
|
| 69 |
-
"Separate
|
| 70 |
-
"
|
| 71 |
],
|
| 72 |
"gui": [
|
| 73 |
"Grounding Intent in Interfaces",
|
|
|
|
| 14 |
"referring": "Locate the target referred to by the following description: ",
|
| 15 |
"dense_point": "Point to: ",
|
| 16 |
"refer_point": "Point to the target referred to by the following description: ",
|
| 17 |
+
"referring_point_in_mask": "Point to the target referred to by the following description: ",
|
| 18 |
+
"dense_point_in_mask": "Point to: ",
|
| 19 |
"layout": "Detect all document layout elements that match the following categories: ",
|
| 20 |
"gui": "Point to the UI element to click for the following instruction: ",
|
| 21 |
}
|
| 22 |
+
CATEGORY_ROUTES = {"grounding", "dense", "layout", "dense_point_in_mask"}
|
| 23 |
|
| 24 |
# Presentation only: never alter the released model prompt or expected answer.
|
| 25 |
TASK_HELP = {
|
|
|
|
| 47 |
"Describe the targets to locate. Points indicate positions, not object boundaries.",
|
| 48 |
"All visible people"
|
| 49 |
],
|
| 50 |
+
"referring_point_in_mask": [
|
| 51 |
+
"Point Inside a Referred Target",
|
| 52 |
+
"Use a description to predict a point inside the target object.",
|
| 53 |
+
"Describe the target. The output is an image-relative point inside the object.",
|
| 54 |
+
"The person wearing a flotation device"
|
| 55 |
],
|
| 56 |
"visual_prompt": [
|
| 57 |
"Grounding by Visual Example",
|
|
|
|
| 65 |
"Use an OCR instruction. Select a transcription to inspect the original pixels.",
|
| 66 |
"Read and locate the text in the image"
|
| 67 |
],
|
| 68 |
+
"dense_point_in_mask": [
|
| 69 |
+
"Point Inside Multiple Targets",
|
| 70 |
+
"Predict an interior point for each matching instance in a crowded scene.",
|
| 71 |
+
"Separate categories with commas. Each prediction marks a point inside an instance.",
|
| 72 |
+
"van, bus, truck, car, pedestrian"
|
| 73 |
],
|
| 74 |
"gui": [
|
| 75 |
"Grounding Intent in Interfaces",
|
grounding_demo/protocol.py
CHANGED
|
@@ -9,7 +9,7 @@ OBJECT_RE = re.compile(
|
|
| 9 |
re.DOTALL,
|
| 10 |
)
|
| 11 |
COORD_RE = re.compile(r"<([0-9]{1,3})>")
|
| 12 |
-
POINT_ROUTES = {"dense_point", "refer_point", "gui"}
|
| 13 |
|
| 14 |
|
| 15 |
@dataclass(frozen=True)
|
|
|
|
| 9 |
re.DOTALL,
|
| 10 |
)
|
| 11 |
COORD_RE = re.compile(r"<([0-9]{1,3})>")
|
| 12 |
+
POINT_ROUTES = {"dense_point", "refer_point", "gui", "referring_point_in_mask", "dense_point_in_mask"}
|
| 13 |
|
| 14 |
|
| 15 |
@dataclass(frozen=True)
|
scripts/build_bundle.py
CHANGED
|
@@ -17,7 +17,9 @@ def main():
|
|
| 17 |
if args.output.exists():
|
| 18 |
raise SystemExit("Output already exists; choose a new archive path.")
|
| 19 |
args.output.parent.mkdir(parents=True, exist_ok=True)
|
| 20 |
-
lock = ROOT / "docs/
|
|
|
|
|
|
|
| 21 |
if not lock.exists():
|
| 22 |
lock = ROOT / "docs/ui_paper_r7_lock.json"
|
| 23 |
if not lock.exists():
|
|
@@ -26,7 +28,8 @@ def main():
|
|
| 26 |
raise ValueError("Original UI/assets/case contract changed; inspect before packaging.")
|
| 27 |
rows = json.loads((ROOT / "data/cases.json").read_text())
|
| 28 |
case_keys = {"id", "route", "route_title", "title", "prompt", "expected", "image",
|
| 29 |
-
"source", "image_sha256", "difficulty", "validation"
|
|
|
|
| 30 |
public_rows = [{k: v for k, v in row.items() if k in case_keys} for row in rows]
|
| 31 |
top = ["app.py", "space_app.py", "README.md", "LICENSE_PENDING.md", "requirements.txt", "constraints-app.txt", ".gitignore", ".env.example",
|
| 32 |
"docs/ui_lock.json", "docs/HF_SPACE_DEPLOYMENT.md", "docs/ZEROGPU_DEPLOYMENT.md",
|
|
|
|
| 17 |
if args.output.exists():
|
| 18 |
raise SystemExit("Output already exists; choose a new archive path.")
|
| 19 |
args.output.parent.mkdir(parents=True, exist_ok=True)
|
| 20 |
+
lock = ROOT / "docs/ui_case_pool_22_lock.json"
|
| 21 |
+
if not lock.exists():
|
| 22 |
+
lock = ROOT / "docs/ui_studio_r8_lock.json"
|
| 23 |
if not lock.exists():
|
| 24 |
lock = ROOT / "docs/ui_paper_r7_lock.json"
|
| 25 |
if not lock.exists():
|
|
|
|
| 28 |
raise ValueError("Original UI/assets/case contract changed; inspect before packaging.")
|
| 29 |
rows = json.loads((ROOT / "data/cases.json").read_text())
|
| 30 |
case_keys = {"id", "route", "route_title", "title", "prompt", "expected", "image",
|
| 31 |
+
"source", "image_sha256", "difficulty", "validation", "image_size",
|
| 32 |
+
"expected_available", "gt_coord", "gt", "visual_prompt"}
|
| 33 |
public_rows = [{k: v for k, v in row.items() if k in case_keys} for row in rows]
|
| 34 |
top = ["app.py", "space_app.py", "README.md", "LICENSE_PENDING.md", "requirements.txt", "constraints-app.txt", ".gitignore", ".env.example",
|
| 35 |
"docs/ui_lock.json", "docs/HF_SPACE_DEPLOYMENT.md", "docs/ZEROGPU_DEPLOYMENT.md",
|
scripts/preflight_model.py
CHANGED
|
@@ -13,7 +13,7 @@ def main():
|
|
| 13 |
parser = argparse.ArgumentParser(description=__doc__)
|
| 14 |
parser.add_argument("model_path", type=Path)
|
| 15 |
parser.add_argument("--metadata-only", action="store_true", help="Do not require weight shards")
|
| 16 |
-
parser.add_argument("--processor", action="store_true", help="Execute reviewed custom processor on all
|
| 17 |
parser.add_argument("--report", type=Path, help="Write a new JSON report (refuse overwrite)")
|
| 18 |
args = parser.parse_args()
|
| 19 |
report = inspect_contract(args.model_path.resolve(), require_weights=not args.metadata_only)
|
|
|
|
| 13 |
parser = argparse.ArgumentParser(description=__doc__)
|
| 14 |
parser.add_argument("model_path", type=Path)
|
| 15 |
parser.add_argument("--metadata-only", action="store_true", help="Do not require weight shards")
|
| 16 |
+
parser.add_argument("--processor", action="store_true", help="Execute reviewed custom processor on all configured cases (CPU)")
|
| 17 |
parser.add_argument("--report", type=Path, help="Write a new JSON report (refuse overwrite)")
|
| 18 |
args = parser.parse_args()
|
| 19 |
report = inspect_contract(args.model_path.resolve(), require_weights=not args.metadata_only)
|
scripts/verify_ui.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
"""
|
| 2 |
import argparse
|
| 3 |
import hashlib
|
| 4 |
import json
|
|
@@ -6,7 +6,8 @@ from pathlib import Path
|
|
| 6 |
|
| 7 |
ROOT = Path(__file__).resolve().parents[1]
|
| 8 |
CASE_KEYS = ["id", "route", "route_title", "title", "prompt", "expected", "image", "source",
|
| 9 |
-
"image_sha256", "difficulty", "validation"
|
|
|
|
| 10 |
|
| 11 |
|
| 12 |
def fingerprint(root):
|
|
@@ -29,7 +30,9 @@ def main():
|
|
| 29 |
parser.add_argument("--reference", type=Path, help="Original project; compare then create lock without overwrite")
|
| 30 |
args = parser.parse_args()
|
| 31 |
current = fingerprint(ROOT)
|
| 32 |
-
lock = ROOT / "docs/
|
|
|
|
|
|
|
| 33 |
if not lock.exists():
|
| 34 |
lock = ROOT / "docs/ui_paper_r7_lock.json"
|
| 35 |
if not lock.exists():
|
|
|
|
| 1 |
+
"""Check the selected release's UI bytes and case contract, excluding backend code."""
|
| 2 |
import argparse
|
| 3 |
import hashlib
|
| 4 |
import json
|
|
|
|
| 6 |
|
| 7 |
ROOT = Path(__file__).resolve().parents[1]
|
| 8 |
CASE_KEYS = ["id", "route", "route_title", "title", "prompt", "expected", "image", "source",
|
| 9 |
+
"image_sha256", "difficulty", "validation", "image_size",
|
| 10 |
+
"expected_available", "gt_coord", "gt", "visual_prompt"]
|
| 11 |
|
| 12 |
|
| 13 |
def fingerprint(root):
|
|
|
|
| 30 |
parser.add_argument("--reference", type=Path, help="Original project; compare then create lock without overwrite")
|
| 31 |
args = parser.parse_args()
|
| 32 |
current = fingerprint(ROOT)
|
| 33 |
+
lock = ROOT / "docs/ui_case_pool_22_lock.json"
|
| 34 |
+
if not lock.exists():
|
| 35 |
+
lock = ROOT / "docs/ui_studio_r8_lock.json"
|
| 36 |
if not lock.exists():
|
| 37 |
lock = ROOT / "docs/ui_paper_r7_lock.json"
|
| 38 |
if not lock.exists():
|
tests/test_bundle.py
CHANGED
|
@@ -1,10 +1,13 @@
|
|
| 1 |
import json
|
|
|
|
|
|
|
| 2 |
import subprocess
|
| 3 |
import sys
|
| 4 |
import tempfile
|
| 5 |
import unittest
|
| 6 |
from pathlib import Path
|
| 7 |
from zipfile import ZipFile
|
|
|
|
| 8 |
|
| 9 |
ROOT = Path(__file__).resolve().parents[1]
|
| 10 |
|
|
@@ -22,8 +25,17 @@ class BundleTests(unittest.TestCase):
|
|
| 22 |
self.assertNotIn("docs/FUYAO_REPRODUCTION.md", names)
|
| 23 |
self.assertNotIn("scripts/service.py", names)
|
| 24 |
rows = json.loads(archive.read("data/cases.json"))
|
| 25 |
-
self.assertEqual(len(rows),
|
| 26 |
self.assertTrue(all("artifact_image_sha256" in row for row in rows))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
payload = archive.read("data/cases.json").decode()
|
| 28 |
for marker in ["/dataset-", "/workspace/", "xiaopeng.com", "source_jsonl", "source_line"]:
|
| 29 |
self.assertNotIn(marker, payload)
|
|
|
|
| 1 |
import json
|
| 2 |
+
import hashlib
|
| 3 |
+
from io import BytesIO
|
| 4 |
import subprocess
|
| 5 |
import sys
|
| 6 |
import tempfile
|
| 7 |
import unittest
|
| 8 |
from pathlib import Path
|
| 9 |
from zipfile import ZipFile
|
| 10 |
+
from PIL import Image
|
| 11 |
|
| 12 |
ROOT = Path(__file__).resolve().parents[1]
|
| 13 |
|
|
|
|
| 25 |
self.assertNotIn("docs/FUYAO_REPRODUCTION.md", names)
|
| 26 |
self.assertNotIn("scripts/service.py", names)
|
| 27 |
rows = json.loads(archive.read("data/cases.json"))
|
| 28 |
+
self.assertEqual(len(rows), 22)
|
| 29 |
self.assertTrue(all("artifact_image_sha256" in row for row in rows))
|
| 30 |
+
self.assertEqual(len({row["route"] for row in rows}), 9)
|
| 31 |
+
for row in rows:
|
| 32 |
+
image_bytes = archive.read(row["image"])
|
| 33 |
+
self.assertEqual(hashlib.sha256(image_bytes).hexdigest(), row["image_sha256"])
|
| 34 |
+
self.assertEqual(row["artifact_image_sha256"], row["image_sha256"])
|
| 35 |
+
with Image.open(BytesIO(image_bytes)) as image:
|
| 36 |
+
self.assertEqual(list(image.size), row["image_size"])
|
| 37 |
+
if not row["expected_available"]:
|
| 38 |
+
self.assertEqual(row["expected"], "")
|
| 39 |
payload = archive.read("data/cases.json").decode()
|
| 40 |
for marker in ["/dataset-", "/workspace/", "xiaopeng.com", "source_jsonl", "source_line"]:
|
| 41 |
self.assertNotIn(marker, payload)
|
tests/test_console.py
CHANGED
|
@@ -26,7 +26,7 @@ class ConsoleTest(unittest.TestCase):
|
|
| 26 |
runs.append(ids[0])
|
| 27 |
self.assertEqual(len(set(runs)), 3)
|
| 28 |
|
| 29 |
-
def
|
| 30 |
for row in app.CASES:
|
| 31 |
with self.subTest(case=row["id"]):
|
| 32 |
query = query_from_prompt(row["prompt"], row["route"])
|
|
@@ -35,6 +35,17 @@ class ConsoleTest(unittest.TestCase):
|
|
| 35 |
def test_category_separator(self):
|
| 36 |
self.assertIn("car</c>person.", compile_query("car, person", "grounding"))
|
| 37 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
def test_case_switch_resets_prediction(self):
|
| 39 |
output = app.load_case(app.CASES[1]["id"])
|
| 40 |
self.assertEqual(output[8], "[]")
|
|
|
|
| 26 |
runs.append(ids[0])
|
| 27 |
self.assertEqual(len(set(runs)), 3)
|
| 28 |
|
| 29 |
+
def test_all_case_prompts_roundtrip_exactly(self):
|
| 30 |
for row in app.CASES:
|
| 31 |
with self.subTest(case=row["id"]):
|
| 32 |
query = query_from_prompt(row["prompt"], row["route"])
|
|
|
|
| 35 |
def test_category_separator(self):
|
| 36 |
self.assertIn("car</c>person.", compile_query("car, person", "grounding"))
|
| 37 |
|
| 38 |
+
def test_every_gallery_route_has_loadable_replacement_examples(self):
|
| 39 |
+
self.assertEqual(len(app.CASES), 22)
|
| 40 |
+
self.assertEqual({row["route"] for row in app.CASES}, set(app.ROUTE_META))
|
| 41 |
+
for route in app.ROUTE_META:
|
| 42 |
+
with self.subTest(route=route):
|
| 43 |
+
choices, gallery, *loaded = app.change_route(route)
|
| 44 |
+
self.assertEqual(len(choices["choices"]), len(app.BY_ROUTE[route]))
|
| 45 |
+
self.assertEqual(len(gallery["value"]), len(app.BY_ROUTE[route]))
|
| 46 |
+
self.assertEqual(loaded[8], "[]")
|
| 47 |
+
self.assertIn("Image ready", loaded[10])
|
| 48 |
+
|
| 49 |
def test_case_switch_resets_prediction(self):
|
| 50 |
output = app.load_case(app.CASES[1]["id"])
|
| 51 |
self.assertEqual(output[8], "[]")
|
tests/test_protocol.py
CHANGED
|
@@ -14,6 +14,14 @@ class ProtocolTest(unittest.TestCase):
|
|
| 14 |
text = "<|object_ref_start|>button<|object_ref_end|><|box_start|><500><600><|box_end|>"
|
| 15 |
self.assertTrue(parse_gam(text, "gui").valid)
|
| 16 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
def test_rejects_wrong_arity(self):
|
| 18 |
text = "<|object_ref_start|>car<|object_ref_end|><|box_start|><1><2><3><|box_end|>"
|
| 19 |
self.assertFalse(parse_gam(text, "grounding").valid)
|
|
|
|
| 14 |
text = "<|object_ref_start|>button<|object_ref_end|><|box_start|><500><600><|box_end|>"
|
| 15 |
self.assertTrue(parse_gam(text, "gui").valid)
|
| 16 |
|
| 17 |
+
def test_mask_point_tasks_parse_points_and_reject_boxes(self):
|
| 18 |
+
point = "<|object_ref_start|>person<|object_ref_end|><|box_start|><500><600><|box_end|>"
|
| 19 |
+
box = "<|object_ref_start|>person<|object_ref_end|><|box_start|><100><200><500><600><|box_end|>"
|
| 20 |
+
for route in ("referring_point_in_mask", "dense_point_in_mask"):
|
| 21 |
+
with self.subTest(route=route):
|
| 22 |
+
self.assertTrue(parse_gam(point, route).valid)
|
| 23 |
+
self.assertFalse(parse_gam(box, route).valid)
|
| 24 |
+
|
| 25 |
def test_rejects_wrong_arity(self):
|
| 26 |
text = "<|object_ref_start|>car<|object_ref_end|><|box_start|><1><2><3><|box_end|>"
|
| 27 |
self.assertFalse(parse_gam(text, "grounding").valid)
|