SolusOps/fatima / notebook_backend /methodology_layout.py
SolusOps's picture
download
raw
13.9 kB
"""Parallel-investigation layout for the recorded study methodology."""
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.patches import Rectangle, Ellipse, FancyArrowPatch
from PIL import Image, ImageDraw, ImageFont, features
INK='#233442'
STAGES=['#386A9D','#A47832','#3B7960','#7A6398','#4D7E87','#247A78']
LANG_COLORS={'english':'#386A9D','hindi':'#C47D22','hinglish':'#6B6F91'}
LABELS={'english':'English','hindi':'Hindi','hinglish':'Roman Hinglish'}
def draw_methodology(base,scores,execution,example,detail):
height=18.9
fig=plt.figure(figsize=(16,height),facecolor='white',dpi=120)
figure_y=lambda top:1-top/height
fig.text(.03,figure_y(.2025),'Study methodology',fontsize=18,weight='bold',color=INK,va='top')
# Inch-based vertical geometry preserves panels 1–3 at their original size.
specifications=[(.03,.405,.94,2.025),(.03,3.105,.435,2.43),(.535,3.105,.435,2.43),
(.03,6.1425,.94,4.3),(.03,10.8425,.94,3.7),(.03,14.9425,.94,3.7)]
bounds=[(x,figure_y(top+h),w,h/height) for x,top,w,h in specifications]
titles=['Research setting','Pretrained VLM','Adapted VLA',
'Existing Evaluation Gaps',
'Lexical Sensitivity under Frozen-Policy Inference',
'Scene-Grounded Lexical Swapping']
axes=[]
for i,(rect,title,color) in enumerate(zip(bounds,titles,STAGES)):
ax=fig.add_axes(rect);ax.set(xlim=(0,1),ylim=(0,1));ax.axis('off')
ax.add_patch(Rectangle((0,0),1,1,facecolor='white',edgecolor=color,linewidth=1.1))
header=.4725/(height*rect[3]);center=1-header/2
ax.add_patch(Rectangle((0,1-header),1,header,facecolor=color,alpha=.10,edgecolor='none'))
cx=.30/(16*rect[2]);dx=.28/(16*rect[2]);dy=.28/(height*rect[3])
ax.add_patch(Ellipse((cx,center),dx,dy,facecolor=color,edgecolor='none'))
ax.text(cx,center,str(i+1),ha='center',va='center',fontsize=12.5,weight='bold',color='white')
ax.text(.62/(16*rect[2]),center,title,va='center',fontsize=15,weight='bold',color=INK)
axes.append(ax)
def text(ax,x,y,s,size=13,bold=False,color=INK,**kw):
return ax.text(x,y,s,fontsize=size,weight='bold' if bold else 'normal',color=color,va='top',**kw)
def hindi(ax,s,x,top,width,image_height,size=13):
assert features.check('raqm'),'Pillow RAQM is required for the exact Hindi label'
rect=ax.get_position();dpi=300
pxw=round(16*rect.width*width*dpi);pxh=round(height*rect.height*image_height*dpi)
font=ImageFont.truetype(str(base/'notebook_backend/fonts/NotoSansDevanagari.ttf'),round(size*dpi/72),layout_engine=ImageFont.Layout.RAQM)
image=Image.new('RGBA',(pxw,pxh),(255,255,255,0));draw=ImageDraw.Draw(image)
box=draw.textbbox((0,0),s,font=font,language='hi',anchor='lt')
assert box[2]<=pxw and box[3]<=pxh,(box,(pxw,pxh))
draw.text((0,0),s,font=font,fill=INK,language='hi',anchor='lt')
ax.imshow(np.asarray(image),extent=(x,x+width,top-image_height,top),aspect='auto',interpolation='lanczos',zorder=2)
def put(ax,x,top,s,size=14,bold=False,color=INK,**kw):
panel_height=ax.get_position().height*height
return text(ax,x,1-top/panel_height,s,size,bold,color,**kw)
# Research context only: robot matching belongs exclusively to branch 3.
ax=axes[0]
ax.imshow(plt.imread(base/'data/backbone/libero_images/task8_state20_agentview_image.png'),
extent=(.035,.1415,.055,.797),aspect='auto',interpolation='lanczos',zorder=2)
text(ax,.175,.69,'LIBERO-Goal',14.5,True,color=STAGES[0])
text(ax,.175,.48,'10 manipulation tasks',12.6)
text(ax,.175,.26,'Example: bowl on plate',12.6)
for lang,x in [('english',.415),('hindi',.615),('hinglish',.805)]:
text(ax,x,.68,LABELS[lang],13,True,color=LANG_COLORS[lang])
if lang=='hindi':hindi(ax,example[lang],x,.455,.18,.17)
else:
instruction=example[lang].replace(' on the plate','\non the plate').replace(' par rakho.','\npar rakho.')
assert ' '.join(instruction.split())==example[lang]
text(ax,x,.45,instruction,13)
# Independent evaluation of the original model; numbers are secondary labels.
ax=axes[1]
text(ax,.045,.815,'Original Qwen3-VL-4B-Instruct',13.5,True,color=STAGES[1])
text(ax,.045,.688,'IndicVisionBench: 98 matched items / language',12.6)
for lang,x in [('english',.045),('hindi',.35),('hinglish',.64)]:
k,n=scores[lang];label='Hinglish' if lang=='hinglish' else LABELS[lang]
text(ax,x,.578,f'{label} {k}/{n}',12,color=LANG_COLORS[lang])
text(ax,.045,.455,'LIBERO: text-only + image-conditioned probes',12.6)
text(ax,.045,.344,'Numbered markers → corrected pink outlines',12.4)
text(ax,.045,.205,'Multilingual instruction understanding',13,True,color=STAGES[1])
text(ax,.045,.095,'Scene recognition remained inconsistent',12.3)
# Matched task IDs, initial states and policy apply to this branch only.
ax=axes[2]
text(ax,.045,.815,'Frozen StarVLA · LIBERO-Goal',13.5,True,color=STAGES[2])
text(ax,.045,.688,'Same task IDs + initial states + adapted policy',12.6)
text(ax,.045,.575,'200 episodes / language · 600 matched outcomes',12.4)
for lang,y in [('english',.415),('hindi',.275),('hinglish',.135)]:
k,n=execution[lang]
text(ax,.045,y+.029,LABELS[lang],12.1,True,color=LANG_COLORS[lang])
ax.add_patch(Rectangle((.345,y-.027),.39,.055,facecolor='#edf0f3',edgecolor='none'))
ax.add_patch(Rectangle((.345,y-.027),.39*k/n,.055,facecolor=LANG_COLORS[lang],edgecolor='none'))
text(ax,.765,y+.029,f'{k}/{n}',12.4)
# Two complementary investigations, explained with the actual audit example.
ax=axes[3]
put(ax,.025,.64,'Existing robot benchmarks',14.5,True,color=STAGES[3])
put(ax,.025,.98,'Standard robot benchmarks',14,True)
put(ax,.025,1.27,'LIBERO, CALVIN, and RoboCasa evaluate whether robots\n'
'can successfully execute task instructions, primarily in English.')
put(ax,.025,1.94,'Instruction robustness',14,True)
put(ax,.025,2.23,'LIBERO-Plus and LIBERO-Para investigate whether robots\n'
'still perform the same task when instructions are reworded\n'
'or their descriptions change.')
put(ax,.025,3.12,'What remains unexplored',14,True,color='#A65353')
put(ax,.025,3.41,'Natural Hindi and Roman Hinglish instruction following is\n'
'not directly evaluated in these original protocols.')
ax.plot([.50,.50],[.045,.865],color='#d6dce1',lw=.8)
put(ax,.525,.64,'Checking Machine Translations',14.5,True,color=STAGES[3])
put(ax,.525,1.02,'10 original English LIBERO instructions were translated\n'
'into Hindi and French using Google Translate, DeepL,\nand Yandex.')
put(ax,.525,1.91,'The 60 resulting translations were inspected for preservation\n'
'of the original objects, actions, and spatial relationships.')
put(ax,.525,2.55,'Back-translation into English was also examined to inspect\n'
'changes in meaning.')
put(ax,.525,3.12,'Original English: “Push the plate”',14,True)
put(ax,.525,3.39,'Yandex Hindi:',14)
hindi(ax,'दबाएं',.64,1-3.36/4.3,.065,.085,size=14)
put(ax,.705,3.39,'(press)',14)
put(ax,.525,3.66,'Changed meaning: “Press the plate”',14,True,color='#A65353')
put(ax,.525,4.00,'Translation can change the physical action requested of the robot.',13.4,True,color=STAGES[3])
# The same intended goal, with only the substituted destination noun emphasized.
ax=axes[4]
put(ax,.025,.61,'Bowl on cabinet · destination-noun substitution · task success',14,True)
for x,label,phrase,count,color in [
(.025,'Original French',"l'armoire.",'0/5','#A65353'),
(.525,'Modified instruction','the cabinet.','3/5',STAGES[2])]:
put(ax,x,.95,label,14,True,color=STAGES[4])
put(ax,x+.405,.94,count,24,True,color=color)
put(ax,x,1.35,'Placez le bol sur le dessus de',14)
put(ax,x+.205,1.35,phrase,14,True,color=color)
put(ax,x,1.70,'English meaning: “Place the bowl on top of the cabinet.”',13.2,
color='#596b78',fontstyle='italic')
ax.plot([.025,.975],[1-2.02/3.7]*2,color='#d6dce1',lw=.8)
put(ax,.025,2.16,'Instruction structure · 20 recorded episodes',13.5,True,color=STAGES[4])
for x,label,count,criterion,example_s in [
(.025,'Keywords only','2/4','Either monitored destination','Bowl cabinet plate'),
(.35,'Target-first','7/8','Requested destination first',
'Bowl ko cabinet ke upar rakho,\nplate par nahi.'),
(.675,'Negation-first / reordered','4/8','Requested destination first',
'Plate par nahi, bowl ko\ncabinet ke upar rakho.')]:
put(ax,x,2.50,label+' · '+count,13,True)
put(ax,x,2.78,criterion,12.3,color='#596b78')
put(ax,x,3.10,example_s,12.3)
# Proposed input intervention, visually distinct from the measured outcomes.
ax=axes[5]
nodes=[(.025,.245),(.30,.325),(.655,.14),(.815,.16)]
for x,width in nodes:
ax.add_patch(Rectangle((x,.04),width,1-.60/3.7-.04,
facecolor='#eff7f6' if x<.65 else '#edf4ef',
edgecolor=STAGES[5],linewidth=.9,linestyle='--' if x==.30 else '-'))
put(ax,.035,.73,'Natural multilingual instructions',13.2,True,color=STAGES[5])
put(ax,.035,1.06,'Hindi',13.4,True,color=LANG_COLORS['hindi'])
hindi(ax,'कैबिनेट की बीच की दराज खोलें',.035,1-1.36/3.7,.225,.105,size=14)
put(ax,.035,1.75,'English: Open the middle drawer\nof the cabinet.',13.2,
color='#596b78',fontstyle='italic')
put(ax,.035,2.38,'Roman Hinglish',13.4,True,color=LANG_COLORS['hinglish'])
put(ax,.035,2.70,'cabinet ka beech wala drawer kholo',13.5)
put(ax,.035,3.05,'English: Open the middle drawer\nof the cabinet.',13.2,
color='#596b78',fontstyle='italic')
put(ax,.31,.73,'Grounded lexical swapping',13.5,True,color=STAGES[5])
put(ax,.31,1.10,'Identify the intended task from the scene,\n'
'then replace selected terms with\npolicy-familiar vocabulary.',13.5)
put(ax,.31,2.04,'Preserve the intended action, objects,\n'
'destinations, and spatial relationships.',13.5)
put(ax,.31,2.83,'Hindi → cabinet ka middle drawer kholo',13.4)
put(ax,.31,3.21,'Roman Hinglish → cabinet ka middle drawer kholo',13.4)
put(ax,.665,.73,'Existing policy',13.5,True,color=STAGES[5])
put(ax,.665,1.65,'StarVLA ·\nQwen3-VL + OFT',13.5)
put(ax,.665,2.55,'Same trained\nweights',13.2)
put(ax,.825,.73,'Expected improvement',12.5,color=STAGES[5])
put(ax,.825,1.65,'Higher multilingual\ntask success',14,True,color=STAGES[5])
for (x,width),(next_x,_) in zip(nodes,nodes[1:]):
ax.add_patch(FancyArrowPatch((x+width+.004,.47),(next_x-.004,.47),arrowstyle='-|>',
mutation_scale=12,color=STAGES[5],linewidth=1.2,shrinkA=0,shrinkB=0))
# Parallel evidence branches converge; complementary auditing belongs to stage 4.
color='#788995'
def line(points):
fig.add_artist(plt.Line2D([p[0] for p in points],[p[1] for p in points],transform=fig.transFigure,color=color,linewidth=1.25))
def arrow(a,b):
fig.add_artist(FancyArrowPatch(a,b,transform=fig.transFigure,arrowstyle='-|>',mutation_scale=14,color=color,linewidth=1.25,shrinkA=0,shrinkB=0))
original_y=lambda value:figure_y((1-value)*13.5)
def original_line(points):line([(x,original_y(v)) for x,v in points])
def original_arrow(a,b):arrow((a[0],original_y(a[1])),(b[0],original_y(b[1])))
original_line([(.5,.814),(.5,.79),(.2475,.79)])
original_line([(.5,.79),(.7525,.79)])
original_arrow((.2475,.79),(.2475,.776));original_arrow((.7525,.79),(.7525,.776))
original_line([(.2475,.584),(.2475,.56),(.7525,.56)])
original_line([(.7525,.584),(.7525,.56)])
original_arrow((.7525,.56),(.7525,.551))
arrow((.5,figure_y(10.52)),(.5,figure_y(10.76)))
arrow((.5,figure_y(14.62)),(.5,figure_y(14.86)))
fig.canvas.draw();renderer=fig.canvas.get_renderer();overflow=[]
for i,ax in enumerate(axes):
box=ax.get_window_extent(renderer)
for artist in ax.texts:
tbox=artist.get_window_extent(renderer)
if tbox.x0<box.x0-1 or tbox.x1>box.x1+1 or tbox.y0<box.y0-1 or tbox.y1>box.y1+1:
overflow.append((i+1,artist.get_text()))
for i,ax in enumerate(axes[3:],start=4):
for j,first in enumerate(ax.texts):
for second in ax.texts[j+1:]:
if first.get_window_extent(renderer).overlaps(second.get_window_extent(renderer)):
overflow.append((i,'Overlapping text: '+first.get_text()+' / '+second.get_text()))
for artist in axes[5].texts[2:]:
x=artist.get_position()[0]
node=next(((left,width) for left,width in nodes if left<=x<left+width),None)
if node:
left,width=node
minimum=axes[5].transData.transform((left,0))[0]
maximum=axes[5].transData.transform((left+width,0))[0]
box=artist.get_window_extent(renderer)
if box.x0<minimum-1 or box.x1>maximum+1:
overflow.append((6,'Text exceeds block: '+artist.get_text()))
assert not overflow,overflow
return fig,{'text_overflow':overflow,'figure_inches':[16,height],
'panel_geometry':[{'x_fraction':x,'top_inches':top,'width_fraction':w,'height_inches':h}
for x,top,w,h in specifications],
'panels_1_to_3_geometry_preserved':True,
'connections':[[1,2],[1,3],[2,4],[3,4],['translation_audit',4],[4,5],[5,6]],
'finding':'Multilingual instruction understanding',
'scene_recognition':'inconsistent','robot_matching_stage':3}

Xet Storage Details

Size:
13.9 kB
·
Xet hash:
74bf85cad7480a14f6e537d73134e8396f60001dc81382a310605494a5922c18

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.