| """Illustrative external instruction adapter. Matplotlib only; no inference.""" | |
| from pathlib import Path | |
| import json | |
| import matplotlib.pyplot as plt | |
| from matplotlib.patches import Rectangle, FancyArrowPatch | |
| from IPython.display import display | |
| BASE=Path(__file__).resolve().parents[1] | |
| TITLE='Proposed Solution: Scene-Grounded Minimal Lexical Adaptation' | |
| INK='#233442' | |
| BLUE,GOLD,TEAL,GREEN='#386A9D','#A47832','#247A78','#3B7960' | |
| def proposal_pipeline_figure(): | |
| with plt.rc_context({'font.family':'DejaVu Sans','svg.fonttype':'none','axes.grid':False}): | |
| fig=plt.figure(figsize=(16,9),dpi=120,facecolor='white') | |
| ax=fig.add_axes([0,0,1,1]);ax.set(xlim=(0,16),ylim=(9,0));ax.axis('off') | |
| texts=[] | |
| def text(x,y,s,size=14,bold=False,color=INK,**kw): | |
| t=ax.text(x,y,s,fontsize=size,weight='bold' if bold else 'normal',color=color,va='top',linespacing=1.4,**kw) | |
| texts.append(t);return t | |
| def box(x,y,w,h,edge,fill='white',dashed=False): | |
| ax.add_patch(Rectangle((x,y),w,h,facecolor=fill,edgecolor=edge,linewidth=1.1,linestyle=(0,(4,3)) if dashed else '-')) | |
| def panel(x,y,w,h,s,color,fill,proposed=False): | |
| box(x,y,w,h,color,dashed=proposed) | |
| ax.add_patch(Rectangle((x,y),w,.52,facecolor=fill,edgecolor='none')) | |
| text(x+.16,y+.13,s,15,True,color) | |
| def arrow(points,color='#526d7e',label=None,pos=None): | |
| if len(points)>2: | |
| xs,ys=zip(*points[:-1]);ax.plot(xs,ys,color=color,lw=1.45,zorder=1) | |
| ax.add_patch(FancyArrowPatch(points[-2],points[-1],arrowstyle='-|>',mutation_scale=16,color=color,lw=1.45,zorder=2)) | |
| if label:text(*pos,label,11.7,color=color,bbox=dict(facecolor='white',edgecolor='none',pad=1.3)) | |
| def sentence(x,y,parts): | |
| for s,color,fill in parts: | |
| t=text(x,y,s,14.2,color=color,bold=fill is not None,bbox=dict(facecolor=fill,edgecolor='none',pad=2) if fill else None) | |
| fig.canvas.draw();x+=t.get_window_extent(fig.canvas.get_renderer()).width/fig.dpi | |
| text(.4,.23,TITLE,21,True) | |
| panel(.4,1.00,4.1,4.20,'1 Instruction + scene',BLUE,'#EAF0F6') | |
| panel(4.88,1.00,4.23,4.20,'2 Ground the intended goal',GOLD,'#F5EFE4',True) | |
| panel(9.49,1.00,6.11,4.20,'3 Language-Balanced Lexical Adaptation',TEAL,'#E5F1F0',True) | |
| text(.62,1.68,'Natural Roman Hinglish',12.8,True,BLUE) | |
| text(.62,1.99,'cabinet ka beech wala drawer kholo',12.4) | |
| translation='[Open the middle drawer of the cabinet.]' | |
| text(.62,2.31,translation,10.8,color='#596b78') | |
| scene=plt.imread(BASE/'data/backbone/libero_images/task0_state20_agentview_image.png') | |
| x,y,side=.64,2.75,2.25 | |
| ax.imshow(scene,extent=(x,x+side,y+side,y),aspect='auto',zorder=3) | |
| ax.add_patch(Rectangle((x+151/224*side,y+82/224*side),72/224*side,125/224*side,edgecolor=GOLD,facecolor='none',linewidth=1.7,zorder=4)) | |
| text(3.04,2.90,'LIBERO\ncamera\nobservation',11.8,color=BLUE) | |
| text(3.04,4.12,'Cabinet\nreference',11.0,color='#596b78') | |
| # Plain input branches join before a single arrowhead enters grounding. | |
| ax.plot([4.05,4.69,4.69],[2.12,2.12,3.10],color='#526d7e',lw=1.45) | |
| ax.plot([4.05,4.69,4.69],[3.87,3.87,3.10],color='#526d7e',lw=1.45) | |
| arrow([(4.69,3.10),(4.86,3.10)]) | |
| # A crop of the actual input scene resolves the drawer by its spatial position. | |
| cx,cy,cw,ch=5.10,1.70,1.64,2.97 | |
| ax.imshow(scene[80:205,155:224],extent=(cx,cx+cw,cy+ch,cy),aspect='auto',zorder=3) | |
| ax.add_patch(Rectangle((cx+14/69*cw,cy+62/125*ch),18/69*cw,21/125*ch, | |
| edgecolor=GOLD,facecolor='none',linewidth=1.8,zorder=4)) | |
| text(6.99,2.85,'Top drawer',12,color='#596b78') | |
| text(6.99,3.42,'Middle drawer',13.2,True,GOLD) | |
| text(6.99,4.01,'Bottom drawer',12,color='#596b78') | |
| arrow([(6.89,3.56),(6.50,3.56),(5.75,3.46)],GOLD) | |
| text(5.10,4.81,'Action: open · Object: drawer',11.8) | |
| text(5.10,5.04,'Fixture: cabinet · Spatial attribute: middle',10.9) | |
| arrow([(9.13,3.10),(9.47,3.10)]) | |
| text(9.72,1.73,'Scene referent → policy-compatible term',12.5,True,TEAL) | |
| text(9.72,2.18,'Original',12.5,color='#596b78') | |
| sentence(9.72,2.48,[('cabinet ka ',INK,None),('beech wala',GOLD,'#F5EFE4'),(' drawer kholo',INK,None)]) | |
| text(9.72,2.81,translation,11.2,color='#596b78') | |
| text(9.72,3.18,'Proposed change',12.5,color='#596b78') | |
| sentence(9.72,3.48,[('cabinet ka ',INK,None),('middle',TEAL,'#E5F1F0'),(' drawer kholo',INK,None)]) | |
| text(9.72,3.81,translation,11.2,color='#596b78') | |
| text(9.72,4.23,'Candidate terms for selective injection',11.9,True) | |
| for xx,yy,s in [(9.72,4.58,'1. Action'),(12.30,4.58,'2. Object'), | |
| (9.72,4.93,'3. Destination'),(12.30,4.93,'4. Spatial relation')]: | |
| text(xx,yy,s,11.8) | |
| panel(.40,6.05,6.80,2.60,'4 Verify semantic preservation',TEAL,'#E5F1F0',True) | |
| panel(8.30,6.05,7.30,2.60,'5 Final outcome',GREEN,'#E7F0E9') | |
| arrow([(12.53,5.23),(12.53,5.65),(4.60,5.65),(4.60,6.02)], | |
| label='Candidate instruction',pos=(9.65,5.39)) | |
| arrow([(6.00,5.23),(6.00,5.40),(3.00,5.40),(3.00,6.02)],GOLD, | |
| 'Grounded goal reference',(3.12,5.32)) | |
| text(3.80,6.96,'Original goal ≡ Adapted goal',18.2,True,TEAL,ha='center') | |
| text(.72,7.81,'Any detected change in original intention\n→ reject candidate',12.8,True,color='#A14747') | |
| arrow([(7.23,7.66),(8.55,7.66)],TEAL) | |
| text(7.77,6.98,'Adapted\ninstruction',10.6,color=TEAL,ha='center') | |
| # Camera observations enter the policy independently of the text verification. | |
| text(10.08,6.61,'Usual camera observations',11.6,color=BLUE,ha='center') | |
| arrow([(10.08,6.89),(10.08,7.04)],BLUE) | |
| box(8.58,7.07,3.00,1.20,GREEN,'#F2F7F3') | |
| text(10.08,7.21,'Existing VLA policy',14,True,GREEN,ha='center') | |
| text(10.08,7.62,'Frozen weights\nUnchanged action head',11.7,ha='center') | |
| text(10.08,8.40,'e.g., StarVLA',10.6,color='#596b78',ha='center') | |
| arrow([(11.61,7.66),(12.08,7.66)],GREEN) | |
| text(12.28,7.20,'Scene-Grounded\nmultilingual\ntask execution',14.3,True,TEAL) | |
| fig.canvas.draw();renderer=fig.canvas.get_renderer();issues=[] | |
| for t in texts: | |
| b=t.get_window_extent(renderer) | |
| if b.x0<0 or b.y0<0 or b.x1>fig.bbox.width or b.y1>fig.bbox.height:issues.append('Outside figure: '+t.get_text()) | |
| for i,t in enumerate(texts): | |
| a=t.get_window_extent(renderer) | |
| for other in texts[i+1:]: | |
| b=other.get_window_extent(renderer) | |
| if min(a.x1,b.x1)-max(a.x0,b.x0)>2 and min(a.y1,b.y1)-max(a.y0,b.y0)>2: | |
| issues.append('Overlapping text: '+t.get_text()+' / '+other.get_text()) | |
| assert not issues,issues | |
| for ext in ['png','svg']:fig.savefig(BASE/'figures'/('proposed_lexical_pipeline.'+ext),dpi=300,facecolor='white') | |
| report=dict(title=TITLE,blocks=5,figure_inches=[16,9],status='Proposed architecture; not experimentally validated', | |
| input_hinglish='cabinet ka beech wala drawer kholo',illustrative_rewrite='cabinet ka middle drawer kholo', | |
| changed_expression=['beech wala','middle'],weights='Frozen weights; unchanged action head', | |
| scene='data/backbone/libero_images/task0_state20_agentview_image.png', | |
| referent_cue='Middle of three cabinet drawers; manually annotated illustration', | |
| layout_issues=issues,svg_text='selectable') | |
| display(fig);plt.close(fig);return report | |
Xet Storage Details
- Size:
- 7.77 kB
- Xet hash:
- 4a3a41d6185c8d3f277cfbe0b6f7c1ced3fa159553e2df87d41d46524be0d113
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.