SolusOps/fatima / notebook_backend /methodology_figure.py
SolusOps's picture
download
raw
5.29 kB
"""Six-stage study design, generated from packaged records without inference."""
from pathlib import Path
import csv
import json
import matplotlib.pyplot as plt
from IPython.display import display
BASE=Path(__file__).resolve().parents[1]
LANGUAGES=['english','hindi','hinglish']
def records():
read=lambda name:json.loads((BASE/name).read_text(encoding='utf-8'))
episodes=read('data/vla/episodes.json')
predictions=[json.loads(line) for line in (BASE/'data/backbone/predictions.jsonl').read_text().splitlines()]
indic=[r for r in predictions if r['dataset']=='IndicVisionBench_parallel']
example={r['condition']:r['instruction'] for r in episodes if r['task_id']==8 and r['initial_state_index']==20}
scores={l:(sum(bool(r['correct']) for r in indic if r['language']==l),sum(r['language']==l for r in indic)) for l in LANGUAGES}
execution={l:(sum(bool(r['success']) for r in episodes if r['condition']==l),sum(r['condition']==l for r in episodes)) for l in LANGUAGES}
assert scores=={'english':(92,98),'hindi':(88,98),'hinglish':(88,98)}
assert execution=={'english':(195,200),'hindi':(58,200),'hinglish':(113,200)}
assert len({r['task_id'] for r in episodes})==10
paired=[{(r['task_id'],r['initial_state_index']) for r in episodes if r['condition']==l} for l in LANGUAGES]
assert paired[0]==paired[1]==paired[2] and len(paired[0])==200
english=example['english']
assert example['hindi']==read('data/instructions/hindi_goal.json')[english]
assert example['hinglish']==read('data/instructions/hinglish_goal.json')[english]
assert len(read('data/french/selected_trials.json'))==38
assert len(read('data/instruction_structure/episodes.json'))==20
with (BASE/'data/translations/translations.csv').open(encoding='utf-8',newline='') as source:
translations=list(csv.DictReader(source))
assert len(translations)==60 and {r['service'] for r in translations}=={'Google','DeepL','Yandex'}
french=read('data/french/selected_trials.json')
originals="Placez le bol sur le dessus de l'armoire."
modified='Placez le bol sur le dessus de the cabinet.'
selected={s:[r for r in french if r['instruction']==s] for s in [originals,modified]}
counts={s:(sum(r['success'] for r in rows),len(rows)) for s,rows in selected.items()}
assert counts=={originals:(0,5),modified:(3,5)}
assert all(sorted(r['initial_state_index'] for r in rows)==list(range(5)) for rows in selected.values())
audit=next(r for r in translations if r['service']=='Yandex' and r['language']=='Hindi' and r['task_id']=='5')
assert audit['translation']=='प्लेट को स्टोव के सामने दबाएं'
structure=read('data/instruction_structure/episodes.json')
prompts=['Bowl cabinet plate','Bowl ko cabinet ke upar rakho, plate par nahi.',
'Plate par nahi, bowl ko cabinet ke upar rakho.']
assert all(any(r['instruction']==s for r in structure) for s in prompts)
structure_counts={s:(sum(r['success'] for r in structure if r['structure']==s),
sum(r['structure']==s for r in structure)) for s in sorted({r['structure'] for r in structure})}
assert structure_counts=={'Keywords only':(2,4),'Normal target-first':(7,8),
'Negation-first / reordered':(4,8)}
natural=read('data/instructions/hinglish_goal.json')['open the middle drawer of the cabinet']
assert natural=='Cabinet ka beech wala drawer kholo.'
detail={'original':originals,'modified':modified,'counts':counts,'translation':audit,
'structure_examples':prompts,'inspected_french_trials':len(french),
'selected_conditions':[r[0]['cohort'] for r in selected.values()],
'structure_counts':structure_counts,'recorded_reference_instruction':natural,
'proposal_inputs':{'hindi':'कैबिनेट की बीच की दराज खोलें',
'hinglish':'cabinet ka beech wala drawer kholo'},
'proposal_input_source':'Author-supplied illustrative examples',
'illustrative_rewrite':'cabinet ka middle drawer kholo',
'adaptation_status':'proposed; rewrite not evaluated'}
return scores,execution,example,detail
def methodology_figure():
from methodology_layout import draw_methodology
scores,execution,example,detail=records()
with plt.rc_context({'font.family':'DejaVu Sans','svg.fonttype':'path','axes.grid':False}):
fig,layout=draw_methodology(BASE,scores,execution,example,detail)
destination=BASE/'figures/study_methodology'
for suffix in ['png','svg']:
fig.savefig(str(destination)+'.'+suffix,dpi=300,bbox_inches='tight',pad_inches=.08,facecolor='white')
report={'stages':6,'indic_scores':scores,'vla_successes':execution,'example_instructions':example,
'example_scene':'data/backbone/libero_images/task8_state20_agentview_image.png',
'translation_records':60,'french_trials':detail['inspected_french_trials'],'structure_episodes':20,
'intervention_evidence':detail,**layout,'exports':['figures/study_methodology.'+s for s in ['png','svg']]}
display(fig);plt.close(fig)
return report

Xet Storage Details

Size:
5.29 kB
·
Xet hash:
2e37dab5e1f47a2a88cc420f94e0993ff73c893e7e67614898395b581534ed00

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.