| """Six-stage study design, generated from packaged records without inference.""" | |
| from pathlib import Path | |
| import csv | |
| import json | |
| import matplotlib.pyplot as plt | |
| from IPython.display import display | |
| BASE=Path(__file__).resolve().parents[1] | |
| LANGUAGES=['english','hindi','hinglish'] | |
| def records(): | |
| read=lambda name:json.loads((BASE/name).read_text(encoding='utf-8')) | |
| episodes=read('data/vla/episodes.json') | |
| predictions=[json.loads(line) for line in (BASE/'data/backbone/predictions.jsonl').read_text().splitlines()] | |
| indic=[r for r in predictions if r['dataset']=='IndicVisionBench_parallel'] | |
| example={r['condition']:r['instruction'] for r in episodes if r['task_id']==8 and r['initial_state_index']==20} | |
| scores={l:(sum(bool(r['correct']) for r in indic if r['language']==l),sum(r['language']==l for r in indic)) for l in LANGUAGES} | |
| execution={l:(sum(bool(r['success']) for r in episodes if r['condition']==l),sum(r['condition']==l for r in episodes)) for l in LANGUAGES} | |
| assert scores=={'english':(92,98),'hindi':(88,98),'hinglish':(88,98)} | |
| assert execution=={'english':(195,200),'hindi':(58,200),'hinglish':(113,200)} | |
| assert len({r['task_id'] for r in episodes})==10 | |
| paired=[{(r['task_id'],r['initial_state_index']) for r in episodes if r['condition']==l} for l in LANGUAGES] | |
| assert paired[0]==paired[1]==paired[2] and len(paired[0])==200 | |
| english=example['english'] | |
| assert example['hindi']==read('data/instructions/hindi_goal.json')[english] | |
| assert example['hinglish']==read('data/instructions/hinglish_goal.json')[english] | |
| assert len(read('data/french/selected_trials.json'))==38 | |
| assert len(read('data/instruction_structure/episodes.json'))==20 | |
| with (BASE/'data/translations/translations.csv').open(encoding='utf-8',newline='') as source: | |
| translations=list(csv.DictReader(source)) | |
| assert len(translations)==60 and {r['service'] for r in translations}=={'Google','DeepL','Yandex'} | |
| french=read('data/french/selected_trials.json') | |
| originals="Placez le bol sur le dessus de l'armoire." | |
| modified='Placez le bol sur le dessus de the cabinet.' | |
| selected={s:[r for r in french if r['instruction']==s] for s in [originals,modified]} | |
| counts={s:(sum(r['success'] for r in rows),len(rows)) for s,rows in selected.items()} | |
| assert counts=={originals:(0,5),modified:(3,5)} | |
| assert all(sorted(r['initial_state_index'] for r in rows)==list(range(5)) for rows in selected.values()) | |
| audit=next(r for r in translations if r['service']=='Yandex' and r['language']=='Hindi' and r['task_id']=='5') | |
| assert audit['translation']=='प्लेट को स्टोव के सामने दबाएं' | |
| structure=read('data/instruction_structure/episodes.json') | |
| prompts=['Bowl cabinet plate','Bowl ko cabinet ke upar rakho, plate par nahi.', | |
| 'Plate par nahi, bowl ko cabinet ke upar rakho.'] | |
| assert all(any(r['instruction']==s for r in structure) for s in prompts) | |
| structure_counts={s:(sum(r['success'] for r in structure if r['structure']==s), | |
| sum(r['structure']==s for r in structure)) for s in sorted({r['structure'] for r in structure})} | |
| assert structure_counts=={'Keywords only':(2,4),'Normal target-first':(7,8), | |
| 'Negation-first / reordered':(4,8)} | |
| natural=read('data/instructions/hinglish_goal.json')['open the middle drawer of the cabinet'] | |
| assert natural=='Cabinet ka beech wala drawer kholo.' | |
| detail={'original':originals,'modified':modified,'counts':counts,'translation':audit, | |
| 'structure_examples':prompts,'inspected_french_trials':len(french), | |
| 'selected_conditions':[r[0]['cohort'] for r in selected.values()], | |
| 'structure_counts':structure_counts,'recorded_reference_instruction':natural, | |
| 'proposal_inputs':{'hindi':'कैबिनेट की बीच की दराज खोलें', | |
| 'hinglish':'cabinet ka beech wala drawer kholo'}, | |
| 'proposal_input_source':'Author-supplied illustrative examples', | |
| 'illustrative_rewrite':'cabinet ka middle drawer kholo', | |
| 'adaptation_status':'proposed; rewrite not evaluated'} | |
| return scores,execution,example,detail | |
| def methodology_figure(): | |
| from methodology_layout import draw_methodology | |
| scores,execution,example,detail=records() | |
| with plt.rc_context({'font.family':'DejaVu Sans','svg.fonttype':'path','axes.grid':False}): | |
| fig,layout=draw_methodology(BASE,scores,execution,example,detail) | |
| destination=BASE/'figures/study_methodology' | |
| for suffix in ['png','svg']: | |
| fig.savefig(str(destination)+'.'+suffix,dpi=300,bbox_inches='tight',pad_inches=.08,facecolor='white') | |
| report={'stages':6,'indic_scores':scores,'vla_successes':execution,'example_instructions':example, | |
| 'example_scene':'data/backbone/libero_images/task8_state20_agentview_image.png', | |
| 'translation_records':60,'french_trials':detail['inspected_french_trials'],'structure_episodes':20, | |
| 'intervention_evidence':detail,**layout,'exports':['figures/study_methodology.'+s for s in ['png','svg']]} | |
| display(fig);plt.close(fig) | |
| return report | |
Xet Storage Details
- Size:
- 5.29 kB
- Xet hash:
- 2e37dab5e1f47a2a88cc420f94e0993ff73c893e7e67614898395b581534ed00
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.