SolusOps/fatima / notebook_backend /comprehension_examples.py
SolusOps's picture
download
raw
6.19 kB
"""Reader-facing examples from preserved comprehension records; no inference."""
from pathlib import Path
import html
import json
import numpy as np
import matplotlib.pyplot as plt
from PIL import Image as PILImage, ImageDraw, ImageFont, features
from IPython.display import display, HTML, Markdown
from editorial_views import finish, panels, COLORS
BASE=Path(__file__).resolve().parents[1]
def _table(columns, rows, widths=None):
"""Native notebook HTML table with wrapped, explicitly separated fields."""
def value(text):
return html.escape(str(text)).replace('\n','<br>')
head=''.join('<th style="text-align:left;vertical-align:top;padding:10px;line-height:1.5">'
+value(c)+'</th>' for c in columns)
body=''.join('<tr>'+''.join(
'<td style="text-align:left;vertical-align:top;white-space:normal;'
'padding:10px;line-height:1.5;overflow-wrap:anywhere;'+
(f'width:{widths[i]}%;' if widths else '')+'">'+value(v)+'</td>'
for i,v in enumerate(row))+'</tr>' for row in rows)
cols=('<colgroup>'+''.join(f'<col style="width:{w}%">' for w in widths)+'</colgroup>') if widths else ''
display(HTML('<table style="width:100%;table-layout:fixed;text-align:left">'+cols+
'<thead><tr>'+head+'</tr></thead><tbody>'+body+'</tbody></table>'))
def text_only_example():
"""One identical item, three languages, with properly shaped Devanagari."""
records=[json.loads(line) for line in
(BASE/'data/backbone/predictions.jsonl').read_text().splitlines()]
order=[('english','English'),('hinglish','Roman Hinglish'),('hindi','Hindi')]
assert features.check('raqm'), 'Hindi rendering requires Pillow RAQM'
dpi=200
width,height=2520,1000
canvas=PILImage.new('RGB',(width,height),'white')
draw=ImageDraw.Draw(canvas)
fontpath=str(BASE/'notebook_backend/fonts/NotoSansDevanagari.ttf')
def font(size,bold=False):
f=ImageFont.truetype(fontpath,size,layout_engine=ImageFont.Layout.RAQM)
if bold:f.set_variation_by_name('Bold')
return f
normal=font(33); bold=font(34,True); heading=font(42,True)
def write(x,y,text,f,fill,maxwidth=730,lineheight=53):
for rawline in text.split('\n'):
line=''
for word in rawline.split(' '):
candidate=(line+' '+word).strip()
if line and draw.textlength(candidate,font=f)>maxwidth:
draw.text((x,y),line,font=f,fill=fill,anchor='lt');y+=lineheight
line=word
else:line=candidate
draw.text((x,y),line,font=f,fill=fill,anchor='lt');y+=lineheight
return y
selected=[]
for i,(language,label) in enumerate(order):
r=next(r for r in records if r['id']=='spatial_1' and r['language']==language)
assert r['answer']=='D'
selected.append(r)
left=18+i*840
color=COLORS[language]
draw.rounded_rectangle((left,8,left+804,height-14),radius=12,
fill='white',outline='#cbd5dc',width=2)
draw.rectangle((left+2,10,left+802,99),fill=color)
write(left+25,29,label,heading,'white')
write(left+25,130,'Exact input',bold,'#233442')
bottom=write(left+25,193,r['prompt'],normal,'#233442')
assert bottom<700, (language,bottom)
draw.line((left+25,721,left+780,721),fill='#cbd5dc',width=2)
write(left+25,748,'Model output: '+r['response'],bold,color)
answer={'english':'D. inside','hinglish':'D. andar','hindi':'D. अंदर'}[language]
write(left+25,811,'Expected: '+answer,normal,'#233442')
correct=r['response']==r['answer']
write(left+25,874,'Result: '+('Correct' if correct else 'Incorrect'),bold,
'#327052' if correct else '#a14747')
fig,ax=plt.subplots(figsize=(width/dpi,height/dpi),layout='constrained')
ax.imshow(np.asarray(canvas));ax.axis('off')
finish(fig,'libero_text_only_triplet')
def outline_examples():
records=json.loads((BASE/'data/visual_exploration/selected_responses.json').read_text())
lookup={r['key']:r for r in records}
def image_paths(key):
return [str(BASE/'data/visual_exploration'/p) for p in lookup[key]['image_files']]
def exact_prompt(prompt):
display(HTML('<p><strong>Exact input prompt</strong></p><blockquote style="white-space:pre-wrap">'
+html.escape(prompt)+'</blockquote>'))
display(Markdown('##### Single-object identification\n\n'
'The same prompt was used with each of these three distinct annotated images. '
'Result assesses the identity of the outlined object.'))
keys=['stage4_agentview_plate','stage4_agentview_wine_bottle','stage4_wrist_wine_bottle']
prompt=lookup[keys[0]]['prompt']
assert all(lookup[k]['prompt']==prompt for k in keys)
exact_prompt(prompt)
titles=['Plate / external camera','Bottle / external camera','Bottle / wrist camera']
panels([image_paths(k)[0] for k in keys],titles,
name='outline_object_inputs',width=12,image_height=3)
_table(['Input image','Expected','Predicted (excerpt)','Result'],[
(titles[0],'Plate','“white, round plate”','Correct'),
(titles[1],'Wine bottle','“dark-colored wine bottle”','Correct'),
(titles[2],'Wine bottle','“ping pong ball”','Incorrect')
],[24,20,40,16])
display(Markdown('##### Category-cued recognition\n\n'
'Result assesses recognition of the food-package category and its position relative to the bowl. '
'The prompt supplies the category names.'))
exact_prompt(lookup['stage2_wrist']['prompt'])
# Use only the supplied wrist image; no second camera was provided for this probe.
panels(image_paths('stage2_wrist'),['Wrist-camera input'],
name='category_cued_input',width=5,image_height=3.3)
_table(['Expected','Predicted (excerpt)','Result'],[
('Packaged food to the right of the bowl',
'“rectangular blue package with a white label”; “to the right of the bowl”','Correct')
],[36,48,16])

Xet Storage Details

Size:
6.19 kB
·
Xet hash:
a994d6824aaa22d33a5ece80c5e7f3f235086c8b7a902a0aa615d63f7f569d03

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.