SolusOps/fatima / notebook_backend /editorial_views.py
SolusOps's picture
download
raw
18.2 kB
"""Native Jupyter exhibits and an independent webpage presentation layer."""
from pathlib import Path
import json
import html
import textwrap
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.patches import FancyBboxPatch, FancyArrowPatch
from IPython.display import display, Markdown, Video, HTML
from presentation import TASKS, LABELS, video_row, matrix_html, highlight_change, esc
COLORS = {'english':'#386A9D', 'hindi':'#C47D22', 'hinglish':'#6B6F91'}
def finish(fig, name):
fig.savefig('figures/'+name+'.png', dpi=220, bbox_inches='tight')
fig.savefig('figures/'+name+'.svg', bbox_inches='tight')
display(fig)
plt.close(fig)
def french_cabinet_summary():
"""README exhibit from the three published five-episode cabinet conditions."""
root=Path(__file__).resolve().parents[1]
rows=json.loads((root/'data/french/selected_trials.json').read_text(encoding='utf-8'))
cohorts=['cabinet_reference','cabinet_destination_noun','cabinet_destination_phrase']
counts=[sum(r['success'] for r in rows if r['cohort']==c) for c in cohorts]
totals=[sum(r['cohort']==c for r in rows) for c in cohorts]
assert counts==[0,3,5] and totals==[5,5,5]
with plt.rc_context({'font.family':'DejaVu Sans','svg.fonttype':'none'}):
fig,ax=plt.subplots(figsize=(10.5,3.1),layout='constrained')
percentages=100*np.array(counts)/totals
ax.barh(range(3),percentages,color=['#8a99a6','#386A9D','#3B7960'],height=.55)
labels=['Original French','Destination noun: the cabinet','Destination phrase: on top of the cabinet']
ax.set(yticks=range(3),yticklabels=labels,xlim=(0,115),xticks=[0,20,40,60,80,100],
xlabel='Recorded task success (%)')
ax.invert_yaxis();ax.grid(axis='y',visible=False)
ax.spines[['top','right']].set_visible(False)
for i,(k,n,p) in enumerate(zip(counts,totals,percentages)):
ax.text(p+1.8,i,f'{k}/{n}',va='center',fontsize=12,fontweight='bold')
for ext in ['png','svg']:
fig.savefig(root/'figures'/('french_cabinet_summary.'+ext),dpi=240,bbox_inches='tight')
display(fig);plt.close(fig)
def panels(paths, titles, notes=None, name='comparison', width=14, image_height=3.1, annotations=None):
notes=notes or ['']*len(paths)
wrap_width=max(20,int(110/len(paths)))
wrapped_notes=['\n'.join(textwrap.fill(line,wrap_width) for line in n.split('\n')) for n in notes]
text_height=max(.25 if not any(notes) else .7, max(len(n.split('\n')) for n in wrapped_notes)*.21+.25)
fig=plt.figure(figsize=(width,image_height+text_height),layout='constrained')
grid=fig.add_gridspec(2,len(paths),height_ratios=[image_height,text_height])
for i,(path,title,note) in enumerate(zip(paths,titles,wrapped_notes)):
ax=fig.add_subplot(grid[0,i]); ax.imshow(plt.imread(path)); ax.axis('off')
if annotations:
# Keep labels in side gutters so they do not obscure simulator objects.
ax.set_xlim(-65,285)
for label,target,position in annotations[i]:
ax.annotate(label,xy=target,xytext=position,xycoords='data',textcoords='data',
ha='right' if position[0]<0 else 'left',va='center',
fontsize=12,fontweight='bold',color='#233442',
bbox=dict(boxstyle='round,pad=.3',facecolor='white',edgecolor='#8d9da7',linewidth=.8),
arrowprops=dict(arrowstyle='-',color='#526d7e',linewidth=1.2,
shrinkA=4,shrinkB=2),annotation_clip=False)
ax.set_title(title,fontsize=11,pad=7)
caption=fig.add_subplot(grid[1,i]);caption.axis('off')
caption.text(0,1,note,va='top',fontsize=11,transform=caption.transAxes)
finish(fig,name)
def metrics(frame, numerator, denominator, name, title, interval=False):
fig,axes=plt.subplots(1,3,figsize=(13.4,2.7),layout='constrained',sharey=True)
for ax,lang in zip(axes,COLORS):
row=frame[frame.language==lang].iloc[0]; k=int(row[numerator]);n=int(row[denominator]);p=100*k/n
ax.barh(0,p,height=.32,color=COLORS[lang]);ax.set(xlim=(0,105),ylim=(-.6,.8),yticks=[],xticks=[0,50,100])
ax.set_title(LABELS[lang],loc='left',color=COLORS[lang],fontsize=12)
ax.text(0,.42,f'{p:.1f}%',fontsize=27,weight='bold')
ax.text(103,.42,f'{k}/{n}',ha='right',fontsize=12)
if interval:
ax.errorbar(p,0,xerr=[[p-row.ci_low],[row.ci_high-p]],fmt='none',capsize=4,color='#253343')
ax.grid(axis='y',visible=False);ax.set_xlabel('Success (%)' if interval else 'Reference accuracy (%)')
for spine in ax.spines.values():spine.set_visible(False)
fig.suptitle(title,fontsize=13)
finish(fig,name)
def vqa_examples(items, paired, ids):
# Native text output preserves Devanagari shaping without a font dependency.
languages=['hindi','english','hinglish']
columns=[]
for key,question_language in zip(ids,languages):
item=items[key]
parts=['<strong>'+LABELS[question_language]+' input</strong><p>'+html.escape(item['variants'][question_language]['question'])+'</p>']
for lang in COLORS:
r=paired[key][lang];letter=r['response'].strip();option=''
if letter in 'ABCD' and len(letter)==1:
option=item['variants']['english']['options']['ABCD'.index(letter)]
line=f"{LABELS[lang]}: {letter} ({option}); {'correct' if r['correct'] else 'incorrect'}"
parts.append('<p>'+html.escape(line)+'</p>')
expected=item['answer']+' ('+item['variants']['english']['options']['ABCD'.index(item['answer'])]+')'
parts.append('<p><strong>Expected:</strong> '+html.escape(expected)+'</p>')
columns.append('<td style="vertical-align:top;width:33.33%;padding:8px 12px;text-align:left">'+''.join(parts)+'</td>')
fig,axes=plt.subplots(1,3,figsize=(12,2.8),layout='constrained')
for i,key in enumerate(ids):
ax=axes[i];ax.imshow(plt.imread('data/backbone/'+items[key]['image']));ax.axis('off')
ax.set_title(['Shared correct answer','English-only correct answer','Hindi and Hinglish correct'][i],fontsize=12)
finish(fig,'indic_three_examples')
display(HTML('<table class="indic-example-inputs" style="width:100%;table-layout:fixed"><tr>'+''.join(columns)+'</tr></table>'))
def native_videos(records, titles=None, width=300):
"""Standard IPython Video renderings in a plain table; no notebook CSS dependency."""
titles=titles or [LABELS.get(r.get('language',r.get('condition')),'French') for r in records]
cells=[]
for r,title in zip(records,titles):
result='' if r.get('success') is None else ('Success' if r['success'] else 'Failure')
video=Video(filename=r['clip'],embed=True,width=width,html_attributes='controls preload="none" playsinline')._repr_html_()
cells.append('<td style="vertical-align:top;text-align:left"><strong>'+html.escape(title)+'</strong><p>'+html.escape(r['instruction'])+'</p>'+video+'<p>'+result+' · <a href="'+r['full_recording']+'">Full recording</a></p></td>')
display(HTML('<table style="width:100%;table-layout:fixed"><tr>'+''.join(cells)+'</tr></table>'))
def contact_sheet(records,tasks,name):
from presentation import select_matrix
selected=select_matrix(records,tasks)
fig,axes=plt.subplots(5,5,figsize=(14,13),layout='constrained')
for ax,r in zip(axes.flat,selected):
ax.imshow(plt.imread(r['poster']));ax.axis('off')
ax.set_title(f"{TASKS[r['task_id']]}\nState {r['initial_state_index']}: {'success' if r['success'] else 'failure'}",fontsize=9,color='#275c73' if r['success'] else '#795242')
finish(fig,name)
for state in range(20,25):
row=[r for r in selected if r['initial_state_index']==state]
videos=[]
for r in row:
v=Video(filename=r['clip'],embed=True,width=180,html_attributes='controls preload="none" playsinline')._repr_html_()
videos.append('<td>'+esc(TASKS[r['task_id']])+v+'<p>'+('Success' if r['success'] else 'Failure')+' · <a href="'+r['full_recording']+'">Full recording</a></p></td>')
display(HTML(f'<details><summary>State {state}: five indexed videos</summary><table style="width:100%;table-layout:fixed"><tr>'+''.join(videos)+'</tr></table></details>'))
def architecture(proposal=False):
if proposal:
labels=['Natural Hindi\nor Hinglish','Semantic\nextraction','Preserve action, object,\ndestination, relation,\nnegation and order','Scene vocabulary +\npolicy-compatible\nalias selection','Minimally modified\ninstruction','Frozen StarVLA\nexecution']
positions=[(.02+i*.167,.42,.135,.38) for i in range(6)]
title='Proposed policy-aligned lexical adaptation (not yet validated)'
else:
labels=['User language\ninstruction','External + wrist\ncamera images','Qwen3-VL-4B-Instruct\nvision-language backbone','StarVLA action queries\n(last hidden states of\naction placeholder tokens)','OFT action head\ncontinuous MLP regression','Predicted action chunk\n8 steps × 7 values','Robot execution\n+ new observations']
positions=[(.02,.65,.18,.25),(.02,.22,.18,.25),(.25,.40,.22,.32),(.52,.40,.22,.32),(.79,.58,.19,.28),(.79,.18,.19,.28),(.49,.03,.23,.22)]
title='Language and two camera views enter a repeated observation-action loop'
fig,ax=plt.subplots(figsize=(16,2.7 if proposal else 5.2),layout='constrained');ax.set(xlim=(0,1),ylim=(.35,.9) if proposal else (0,1));ax.axis('off')
for i,(label,(x,y,w,h)) in enumerate(zip(labels,positions)):
ax.add_patch(FancyBboxPatch((x,y),w,h,boxstyle='round,pad=.008,rounding_size=.012',facecolor='#edf3f8' if i>1 else '#f5f7f9',edgecolor='#8ba4b8',linewidth=1.1))
ax.text(x+w/2,y+h/2,label,ha='center',va='center',fontsize=11)
def arrow(a,b,rad=0):ax.add_patch(FancyArrowPatch(a,b,arrowstyle='-|>',mutation_scale=15,color='#386A9D',linewidth=1.5,connectionstyle=f'arc3,rad={rad}'))
if proposal:
for p,q in zip(positions,positions[1:]):arrow((p[0]+p[2]+.008,.61),(q[0]-.008,.61))
else:
arrow((.21,.77),(.24,.65));arrow((.21,.35),(.24,.46));arrow((.48,.56),(.51,.56))
arrow((.75,.64),(.78,.72));arrow((.885,.57),(.885,.47));arrow((.78,.29),(.73,.17));arrow((.49,.13),(.11,.20),-.2)
ax.text(.285,.09,'Subsequent observations',fontsize=10,color='#386A9D',ha='center')
ax.set_title(title,fontsize=13,pad=15);finish(fig,'lexical_adapter_architecture' if proposal else 'vla_architecture')
def full_table(frame):
if 'recording' in frame.columns:
formats={c:lambda x:html.escape(str(x)) for c in frame.columns}
formats['recording']=lambda p:('<a href="'+html.escape(str(p),quote=True)+'">Full recording</a>') if p else ''
display(HTML(frame.to_html(index=False,formatters=formats,escape=False)))
else:
with pd.option_context('display.max_rows',None,'display.max_columns',None,'display.max_colwidth',None):display(frame)
def exact_records(records, keys=('prompt','response'), label='id'):
"""Exact strings, without punctuation normalization or response truncation."""
for r in records:
body='#### '+str(r.get(label,r.get('key','Record')))+'\n\n'
body+=' · '.join(str(r.get(k,'')) for k in ['language','dataset','mode'] if r.get(k))+'\n\n'
for key in keys:
if key in r:body+='**'+key+'**\n\n````text\n'+str(r[key])+'\n````\n\n'
display(Markdown(body))
WEB_CSS='''<style>
html{scroll-behavior:smooth}body.jp-Notebook{background:#fff;color:#243443;padding:0!important;font-family:Cambria,Georgia,serif!important;font-size:17px}
main{max-width:1460px;margin:0 auto;padding:36px 4vw 80px}.jp-Notebook{max-width:none!important}
.jp-Cell{padding:0!important;margin:0!important}.jp-Cell-inputWrapper,.jp-Cell-outputWrapper{width:100%}.jp-InputPrompt,.jp-OutputPrompt{display:none!important}
.jp-RenderedHTMLCommon{font-family:Cambria,Georgia,serif!important;font-size:17px!important;line-height:1.65!important}
.jp-RenderedHTMLCommon p,.jp-RenderedHTMLCommon ul,.jp-RenderedHTMLCommon ol{max-width:880px}
.jp-RenderedHTMLCommon h1{font-size:clamp(30px,3.3vw,49px);line-height:1.15;max-width:1040px;margin:24px 0!important;color:#163f61}
.jp-RenderedHTMLCommon h2{font-size:30px;border-top:3px solid #386A9D;padding-top:22px;margin-top:66px!important;color:#163f61}
.jp-RenderedHTMLCommon h3{font-size:23px;margin-top:36px!important}.jp-RenderedHTMLCommon h4{font-size:19px}
.jp-OutputArea-output{overflow-x:auto!important}.jp-RenderedImage img{max-width:100%;height:auto}.jp-RenderedHTMLCommon table{font-family:system-ui,sans-serif;font-size:14px;line-height:1.5;white-space:normal!important;width:100%}
.jp-RenderedHTMLCommon td,.jp-RenderedHTMLCommon th{padding:10px 12px!important;vertical-align:top!important;text-align:left!important}
.research{font-family:system-ui,sans-serif;line-height:1.5}.research video{aspect-ratio:1;width:100%;background:#f2f5f8}
.research .comparison{display:grid;grid-template-columns:repeat(auto-fit,minmax(240px,1fr));gap:24px}
.research .media-comparison:has(>.video-card:only-child){grid-template-columns:minmax(0,440px)}
.research .matrix-scroll{overflow:visible}.research .rollout-matrix{display:grid;grid-template-columns:repeat(5,minmax(0,1fr));min-width:0;gap:16px}
.research .matrix-heading{font-size:14px;border-bottom:2px solid #9badbb;padding-bottom:10px}.research .matrix-heading p{font-size:13px}
.research .video-card{min-width:0;border-bottom:1px solid #dbe2e8;padding-bottom:14px}.research .video-meta{font-size:12px;display:flex;justify-content:space-between;gap:6px}
.research .instruction{min-height:3em;font-size:15px}.research .outcome{font-weight:700}.research .success{color:#275c73}.research .failure{color:#795242}
.research .video-card>a{font-size:12px}.research mark{background:#e2edf6;color:#163f61;font-weight:bold}
nav{font-family:system-ui,sans-serif;display:flex;flex-wrap:wrap;gap:12px 24px;padding:16px 0;border-bottom:1px solid #bccbd7;font-size:14px}a{color:#245e89}a:focus-visible,summary:focus-visible{outline:3px solid #c47d22;outline-offset:3px}
@media(max-width:800px){main{padding:20px 18px 50px}.research .matrix-heading{display:none}.research .rollout-matrix{grid-template-columns:repeat(2,minmax(0,1fr))}.jp-RenderedHTMLCommon{font-size:16px!important}.jp-RenderedHTMLCommon h2{font-size:25px}.research .comparison{grid-template-columns:1fr}.jp-RenderedImage img{max-width:none;min-width:1000px}.jp-OutputArea-output{min-width:0;max-width:100%;overflow-x:auto!important}}
@media(max-width:430px){.research .rollout-matrix{grid-template-columns:1fr}}
@media(prefers-reduced-motion:reduce){html{scroll-behavior:auto}}
</style>'''
def web_exhibit(kind, evidence):
if kind=='matrix':return matrix_html(evidence['matrix_media'],[0,2,4,6,8])
groups={'intro':'ordinary_success','matched':'matched_language_contrast','unusual':'unusual_success'}
if kind in groups:
records=[r for r in evidence['media'] if r['group']==groups[kind]]
titles={'intro':['Example: Hindi bowl placement'],'unusual':['Hindi: cream cheese in bowl']}.get(kind)
return video_row(records,titles)
if kind=='behavior':
records=[next(r for r in evidence['media'] if r['group']=='recurring_bowl_to_stove' and r['language']=='english'),next(r for r in evidence['media'] if r['id']=='case_french_cabinet_baseline')]
return video_row(records,['Greeting: Hello.','Requested: bowl on cabinet'])
if kind.startswith('french_'):
lookup={'french_middle':('middle_drawer_codeswitch',['A_fully_french','B_english_object_only']),
'french_plate':('plate_stove_ADGH_stop_first_success',['A_verb_only','D_verb_object']),
'french_top':('top_drawer_bowl_conditions_3_4_five',['C_english_destination_only','D_english_object_and_destination'])}
prefix,conditions=lookup[kind];rows=[next(r for r in evidence['media'] if r.get('run_id','').startswith(prefix) and r.get('condition')==c) for c in conditions]
return '<div class="research"><p>'+esc(rows[0]['instruction'])+' → '+highlight_change(rows[0]['instruction'],rows[1]['instruction'])+'</p></div>'+video_row(rows,['Original / comparison','Modified instruction'])
raise ValueError(kind)
def benchmark_gallery(corpus, predictions):
"""Every evaluated IndicVisionBench image, with saved multilingual outcomes."""
import base64, io
from PIL import Image as PILImage
examples=[r for r in corpus if r['dataset']=='IndicVisionBench_parallel']
lookup={(r['id'],r['language']):r for r in predictions}
assert len(examples)==98
for start in range(0,len(examples),3):
cards=[]
for item in examples[start:start+3]:
with PILImage.open(Path('data/backbone')/item['image']) as original:
thumbnail=original.convert('RGB');thumbnail.thumbnail((500,320))
buffer=io.BytesIO();thumbnail.save(buffer,format='PNG')
image_uri='data:image/png;base64,'+base64.b64encode(buffer.getvalue()).decode('ascii')
content='<strong>'+html.escape(item['id'])+'</strong><br><img src="'+image_uri+'" style="max-width:100%;height:200px;object-fit:contain" alt="'+html.escape(item['id'])+'"><p>'+html.escape(item['variants']['english']['question'])+'</p>'
content+='<p><strong>Reference: '+html.escape(item['answer'])+'</strong> · '+html.escape(item['variants']['english']['options'][ord(item['answer'])-65])+'</p>'
for language in ['english','hindi','hinglish']:
record=lookup[item['id'],language]
content+='<div>'+html.escape(LABELS[language])+': '+html.escape(record['response'])+' · '+('Correct' if record['correct'] else 'Incorrect')+'</div>'
content+='<a href="data/backbone/'+html.escape(item['image'])+'">Full-resolution image</a>'
cards.append('<td style="width:33.33%;vertical-align:top;text-align:left;padding:12px;border:1px solid #dbe2e8">'+content+'</td>')
display(HTML('<table style="table-layout:fixed;width:100%;font-size:14px"><tr>'+''.join(cards)+'</tr></table>'))

Xet Storage Details

Size:
18.2 kB
·
Xet hash:
0693dd81a4a31a19b497358cc16111e02e0e073645add0c3a64f3055dee29133

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.