SolusOps/fatima / notebook_backend /proposal_pipeline.py
SolusOps's picture
download
raw
7.77 kB
"""Illustrative external instruction adapter. Matplotlib only; no inference."""
from pathlib import Path
import json
import matplotlib.pyplot as plt
from matplotlib.patches import Rectangle, FancyArrowPatch
from IPython.display import display
BASE=Path(__file__).resolve().parents[1]
TITLE='Proposed Solution: Scene-Grounded Minimal Lexical Adaptation'
INK='#233442'
BLUE,GOLD,TEAL,GREEN='#386A9D','#A47832','#247A78','#3B7960'
def proposal_pipeline_figure():
with plt.rc_context({'font.family':'DejaVu Sans','svg.fonttype':'none','axes.grid':False}):
fig=plt.figure(figsize=(16,9),dpi=120,facecolor='white')
ax=fig.add_axes([0,0,1,1]);ax.set(xlim=(0,16),ylim=(9,0));ax.axis('off')
texts=[]
def text(x,y,s,size=14,bold=False,color=INK,**kw):
t=ax.text(x,y,s,fontsize=size,weight='bold' if bold else 'normal',color=color,va='top',linespacing=1.4,**kw)
texts.append(t);return t
def box(x,y,w,h,edge,fill='white',dashed=False):
ax.add_patch(Rectangle((x,y),w,h,facecolor=fill,edgecolor=edge,linewidth=1.1,linestyle=(0,(4,3)) if dashed else '-'))
def panel(x,y,w,h,s,color,fill,proposed=False):
box(x,y,w,h,color,dashed=proposed)
ax.add_patch(Rectangle((x,y),w,.52,facecolor=fill,edgecolor='none'))
text(x+.16,y+.13,s,15,True,color)
def arrow(points,color='#526d7e',label=None,pos=None):
if len(points)>2:
xs,ys=zip(*points[:-1]);ax.plot(xs,ys,color=color,lw=1.45,zorder=1)
ax.add_patch(FancyArrowPatch(points[-2],points[-1],arrowstyle='-|>',mutation_scale=16,color=color,lw=1.45,zorder=2))
if label:text(*pos,label,11.7,color=color,bbox=dict(facecolor='white',edgecolor='none',pad=1.3))
def sentence(x,y,parts):
for s,color,fill in parts:
t=text(x,y,s,14.2,color=color,bold=fill is not None,bbox=dict(facecolor=fill,edgecolor='none',pad=2) if fill else None)
fig.canvas.draw();x+=t.get_window_extent(fig.canvas.get_renderer()).width/fig.dpi
text(.4,.23,TITLE,21,True)
panel(.4,1.00,4.1,4.20,'1 Instruction + scene',BLUE,'#EAF0F6')
panel(4.88,1.00,4.23,4.20,'2 Ground the intended goal',GOLD,'#F5EFE4',True)
panel(9.49,1.00,6.11,4.20,'3 Language-Balanced Lexical Adaptation',TEAL,'#E5F1F0',True)
text(.62,1.68,'Natural Roman Hinglish',12.8,True,BLUE)
text(.62,1.99,'cabinet ka beech wala drawer kholo',12.4)
translation='[Open the middle drawer of the cabinet.]'
text(.62,2.31,translation,10.8,color='#596b78')
scene=plt.imread(BASE/'data/backbone/libero_images/task0_state20_agentview_image.png')
x,y,side=.64,2.75,2.25
ax.imshow(scene,extent=(x,x+side,y+side,y),aspect='auto',zorder=3)
ax.add_patch(Rectangle((x+151/224*side,y+82/224*side),72/224*side,125/224*side,edgecolor=GOLD,facecolor='none',linewidth=1.7,zorder=4))
text(3.04,2.90,'LIBERO\ncamera\nobservation',11.8,color=BLUE)
text(3.04,4.12,'Cabinet\nreference',11.0,color='#596b78')
# Plain input branches join before a single arrowhead enters grounding.
ax.plot([4.05,4.69,4.69],[2.12,2.12,3.10],color='#526d7e',lw=1.45)
ax.plot([4.05,4.69,4.69],[3.87,3.87,3.10],color='#526d7e',lw=1.45)
arrow([(4.69,3.10),(4.86,3.10)])
# A crop of the actual input scene resolves the drawer by its spatial position.
cx,cy,cw,ch=5.10,1.70,1.64,2.97
ax.imshow(scene[80:205,155:224],extent=(cx,cx+cw,cy+ch,cy),aspect='auto',zorder=3)
ax.add_patch(Rectangle((cx+14/69*cw,cy+62/125*ch),18/69*cw,21/125*ch,
edgecolor=GOLD,facecolor='none',linewidth=1.8,zorder=4))
text(6.99,2.85,'Top drawer',12,color='#596b78')
text(6.99,3.42,'Middle drawer',13.2,True,GOLD)
text(6.99,4.01,'Bottom drawer',12,color='#596b78')
arrow([(6.89,3.56),(6.50,3.56),(5.75,3.46)],GOLD)
text(5.10,4.81,'Action: open · Object: drawer',11.8)
text(5.10,5.04,'Fixture: cabinet · Spatial attribute: middle',10.9)
arrow([(9.13,3.10),(9.47,3.10)])
text(9.72,1.73,'Scene referent → policy-compatible term',12.5,True,TEAL)
text(9.72,2.18,'Original',12.5,color='#596b78')
sentence(9.72,2.48,[('cabinet ka ',INK,None),('beech wala',GOLD,'#F5EFE4'),(' drawer kholo',INK,None)])
text(9.72,2.81,translation,11.2,color='#596b78')
text(9.72,3.18,'Proposed change',12.5,color='#596b78')
sentence(9.72,3.48,[('cabinet ka ',INK,None),('middle',TEAL,'#E5F1F0'),(' drawer kholo',INK,None)])
text(9.72,3.81,translation,11.2,color='#596b78')
text(9.72,4.23,'Candidate terms for selective injection',11.9,True)
for xx,yy,s in [(9.72,4.58,'1. Action'),(12.30,4.58,'2. Object'),
(9.72,4.93,'3. Destination'),(12.30,4.93,'4. Spatial relation')]:
text(xx,yy,s,11.8)
panel(.40,6.05,6.80,2.60,'4 Verify semantic preservation',TEAL,'#E5F1F0',True)
panel(8.30,6.05,7.30,2.60,'5 Final outcome',GREEN,'#E7F0E9')
arrow([(12.53,5.23),(12.53,5.65),(4.60,5.65),(4.60,6.02)],
label='Candidate instruction',pos=(9.65,5.39))
arrow([(6.00,5.23),(6.00,5.40),(3.00,5.40),(3.00,6.02)],GOLD,
'Grounded goal reference',(3.12,5.32))
text(3.80,6.96,'Original goal ≡ Adapted goal',18.2,True,TEAL,ha='center')
text(.72,7.81,'Any detected change in original intention\n→ reject candidate',12.8,True,color='#A14747')
arrow([(7.23,7.66),(8.55,7.66)],TEAL)
text(7.77,6.98,'Adapted\ninstruction',10.6,color=TEAL,ha='center')
# Camera observations enter the policy independently of the text verification.
text(10.08,6.61,'Usual camera observations',11.6,color=BLUE,ha='center')
arrow([(10.08,6.89),(10.08,7.04)],BLUE)
box(8.58,7.07,3.00,1.20,GREEN,'#F2F7F3')
text(10.08,7.21,'Existing VLA policy',14,True,GREEN,ha='center')
text(10.08,7.62,'Frozen weights\nUnchanged action head',11.7,ha='center')
text(10.08,8.40,'e.g., StarVLA',10.6,color='#596b78',ha='center')
arrow([(11.61,7.66),(12.08,7.66)],GREEN)
text(12.28,7.20,'Scene-Grounded\nmultilingual\ntask execution',14.3,True,TEAL)
fig.canvas.draw();renderer=fig.canvas.get_renderer();issues=[]
for t in texts:
b=t.get_window_extent(renderer)
if b.x0<0 or b.y0<0 or b.x1>fig.bbox.width or b.y1>fig.bbox.height:issues.append('Outside figure: '+t.get_text())
for i,t in enumerate(texts):
a=t.get_window_extent(renderer)
for other in texts[i+1:]:
b=other.get_window_extent(renderer)
if min(a.x1,b.x1)-max(a.x0,b.x0)>2 and min(a.y1,b.y1)-max(a.y0,b.y0)>2:
issues.append('Overlapping text: '+t.get_text()+' / '+other.get_text())
assert not issues,issues
for ext in ['png','svg']:fig.savefig(BASE/'figures'/('proposed_lexical_pipeline.'+ext),dpi=300,facecolor='white')
report=dict(title=TITLE,blocks=5,figure_inches=[16,9],status='Proposed architecture; not experimentally validated',
input_hinglish='cabinet ka beech wala drawer kholo',illustrative_rewrite='cabinet ka middle drawer kholo',
changed_expression=['beech wala','middle'],weights='Frozen weights; unchanged action head',
scene='data/backbone/libero_images/task0_state20_agentview_image.png',
referent_cue='Middle of three cabinet drawers; manually annotated illustration',
layout_issues=issues,svg_text='selectable')
display(fig);plt.close(fig);return report

Xet Storage Details

Size:
7.77 kB
·
Xet hash:
4a3a41d6185c8d3f277cfbe0b6f7c1ced3fa159553e2df87d41d46524be0d113

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.