| """Load only the curated public evidence. No model, simulator or research archive access.""" | |
| from pathlib import Path | |
| import json | |
| import numpy as np | |
| import pandas as pd | |
| from analysis import parse_choice,validate_pairs,paired_difference,wilson,paired_items,contingency | |
| from presentation import LABELS | |
| DISPLAY_CASES=[{'id': 'case_matched_t04_s20_english', 'group': 'matched_language_contrast', 'task_id': 4, 'initial_state_index': 20, 'instruction': 'put the bowl on top of the cabinet', 'clip': 'media/layout/case_matched_t04_s20_english.mp4', 'poster': 'media/layout/case_matched_t04_s20_english_final_poster.png', 'success': True, 'language': 'english', 'full_recording': 'media/layout/full/case_matched_t04_s20_english.mp4'}, {'id': 'case_matched_t04_s20_hindi', 'group': 'matched_language_contrast', 'task_id': 4, 'initial_state_index': 20, 'instruction': 'बाउल को कैबिनेट के ऊपर रखो।', 'clip': 'media/layout/case_matched_t04_s20_hindi.mp4', 'poster': 'media/layout/case_matched_t04_s20_hindi_final_poster.png', 'success': False, 'language': 'hindi', 'full_recording': 'media/layout/full/case_matched_t04_s20_hindi.mp4'}, {'id': 'case_matched_t04_s20_hinglish', 'group': 'matched_language_contrast', 'task_id': 4, 'initial_state_index': 20, 'instruction': 'Bowl ko cabinet ke upar rakho.', 'clip': 'media/layout/case_matched_t04_s20_hinglish.mp4', 'poster': 'media/layout/case_matched_t04_s20_hinglish_final_poster.png', 'success': True, 'language': 'hinglish', 'full_recording': 'media/layout/full/case_matched_t04_s20_hinglish.mp4'}, {'id': 'case_hindi_t08_s20_success', 'group': 'ordinary_success', 'task_id': 8, 'initial_state_index': 20, 'instruction': 'बाउल को प्लेट पर रखो।', 'clip': 'media/layout/case_hindi_t08_s20_success.mp4', 'poster': 'media/layout/case_hindi_t08_s20_success_final_poster.png', 'success': True, 'language': 'hindi', 'full_recording': 'media/layout/full/case_hindi_t08_s20_success.mp4'}, {'id': 'case_unusual_hindi_t06_s01', 'group': 'unusual_success', 'task_id': 6, 'initial_state_index': 1, 'instruction': 'क्रीम चीज़ को बाउल में रखो।', 'clip': 'media/layout/case_unusual_hindi_t06_s01.mp4', 'poster': 'media/layout/case_unusual_hindi_t06_s01_final_poster.png', 'success': True, 'language': 'hindi', 'full_recording': 'media/layout/full/case_unusual_hindi_t06_s01.mp4'}, {'id': 'case_greeting_english_t02_s00', 'group': 'recurring_bowl_to_stove', 'task_id': 2, 'initial_state_index': 0, 'instruction': 'Hello.', 'clip': 'media/layout/case_greeting_english_t02_s00.mp4', 'poster': 'media/layout/case_greeting_english_t02_s00_final_poster.png', 'success': None, 'language': 'english', 'full_recording': 'media/layout/full/case_greeting_english_t02_s00.mp4'}, {'id': 'case_french_cabinet_baseline', 'group': 'french_illustrative_contrast', 'task_id': 4, 'initial_state_index': 0, 'instruction': "Placez le bol sur le dessus de l'armoire.", 'clip': 'media/layout/case_french_cabinet_baseline.mp4', 'poster': 'media/layout/case_french_cabinet_baseline_final_poster.png', 'success': False, 'language': 'french', 'full_recording': 'media/layout/full/case_french_cabinet_baseline.mp4'}, {'id': 'french_selection_28', 'group': 'french_lexical', 'task_id': 0, 'initial_state_index': 0, 'instruction': "Ouvrez le tiroir du milieu de l'armoire.", 'clip': 'media/layout/french_selection_28.mp4', 'poster': 'media/layout/french_selection_28_final_poster.png', 'success': False, 'language': 'french', 'condition': 'A_fully_french', 'full_recording': 'media/layout/full/french_selection_28.mp4', 'run_id': 'middle_drawer_codeswitch'}, {'id': 'french_selection_29', 'group': 'french_lexical', 'task_id': 0, 'initial_state_index': 0, 'instruction': "Ouvrez the middle drawer de l'armoire.", 'clip': 'media/layout/french_selection_29.mp4', 'poster': 'media/layout/french_selection_29_final_poster.png', 'success': True, 'language': 'french', 'condition': 'B_english_object_only', 'full_recording': 'media/layout/full/french_selection_29.mp4', 'run_id': 'middle_drawer_codeswitch'}, {'id': 'french_selection_31', 'group': 'french_lexical', 'task_id': 5, 'initial_state_index': 0, 'instruction': "Push l'assiette vers l'avant de la cuisinière.", 'clip': 'media/layout/french_selection_31.mp4', 'poster': 'media/layout/french_selection_31_final_poster.png', 'success': False, 'language': 'french', 'condition': 'A_verb_only', 'full_recording': 'media/layout/full/french_selection_31.mp4', 'run_id': 'plate_stove_ADGH_stop_first_success'}, {'id': 'french_selection_32', 'group': 'french_lexical', 'task_id': 5, 'initial_state_index': 0, 'instruction': "Push the plate vers l'avant de la cuisinière.", 'clip': 'media/layout/french_selection_32.mp4', 'poster': 'media/layout/french_selection_32_final_poster.png', 'success': True, 'language': 'french', 'condition': 'D_verb_object', 'full_recording': 'media/layout/full/french_selection_32.mp4', 'run_id': 'plate_stove_ADGH_stop_first_success'}, {'id': 'french_selection_44', 'group': 'french_lexical', 'task_id': 3, 'initial_state_index': 0, 'instruction': "Ouvrez the top drawer et placez le bol à l'intérieur.", 'clip': 'media/layout/french_selection_44.mp4', 'poster': 'media/layout/french_selection_44_final_poster.png', 'success': False, 'language': 'french', 'condition': 'C_english_destination_only', 'full_recording': 'media/layout/full/french_selection_44.mp4', 'run_id': 'top_drawer_bowl_conditions_3_4_five'}, {'id': 'french_selection_49', 'group': 'french_lexical', 'task_id': 3, 'initial_state_index': 0, 'instruction': "Ouvrez the top drawer et placez the bowl à l'intérieur.", 'clip': 'media/layout/french_selection_49.mp4', 'poster': 'media/layout/french_selection_49_final_poster.png', 'success': True, 'language': 'french', 'condition': 'D_english_object_and_destination', 'full_recording': 'media/layout/full/french_selection_49.mp4', 'run_id': 'top_drawer_bowl_conditions_3_4_five'}] | |
| MATRIX_MEDIA=[{'task_id': 1, 'initial_state_index': 20, 'condition': 'hinglish', 'instruction': 'Bowl ko stove par rakho.', 'success': True, 'clip': 'media/layout/hinglish_t01_s20.mp4', 'poster': 'media/layout/hinglish_t01_s20_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task01_state20.mp4'}, {'task_id': 3, 'initial_state_index': 20, 'condition': 'hinglish', 'instruction': 'Cabinet ka sabse upar wala drawer kholo aur bowl ko uske andar rakho.', 'success': False, 'clip': 'media/layout/hinglish_t03_s20.mp4', 'poster': 'media/layout/hinglish_t03_s20_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task03_state20.mp4'}, {'task_id': 5, 'initial_state_index': 20, 'condition': 'hinglish', 'instruction': 'Plate ko stove ke saamne dhakelo.', 'success': False, 'clip': 'media/layout/hinglish_t05_s20.mp4', 'poster': 'media/layout/hinglish_t05_s20_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task05_state20.mp4'}, {'task_id': 7, 'initial_state_index': 20, 'condition': 'hinglish', 'instruction': 'Stove chalu karo.', 'success': True, 'clip': 'media/layout/hinglish_t07_s20.mp4', 'poster': 'media/layout/hinglish_t07_s20_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task07_state20.mp4'}, {'task_id': 9, 'initial_state_index': 20, 'condition': 'hinglish', 'instruction': 'Wine bottle ko wine rack par rakho.', 'success': True, 'clip': 'media/layout/hinglish_t09_s20.mp4', 'poster': 'media/layout/hinglish_t09_s20_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task09_state20.mp4'}, {'task_id': 1, 'initial_state_index': 21, 'condition': 'hinglish', 'instruction': 'Bowl ko stove par rakho.', 'success': True, 'clip': 'media/layout/hinglish_t01_s21.mp4', 'poster': 'media/layout/hinglish_t01_s21_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task01_state21.mp4'}, {'task_id': 3, 'initial_state_index': 21, 'condition': 'hinglish', 'instruction': 'Cabinet ka sabse upar wala drawer kholo aur bowl ko uske andar rakho.', 'success': False, 'clip': 'media/layout/hinglish_t03_s21.mp4', 'poster': 'media/layout/hinglish_t03_s21_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task03_state21.mp4'}, {'task_id': 5, 'initial_state_index': 21, 'condition': 'hinglish', 'instruction': 'Plate ko stove ke saamne dhakelo.', 'success': False, 'clip': 'media/layout/hinglish_t05_s21.mp4', 'poster': 'media/layout/hinglish_t05_s21_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task05_state21.mp4'}, {'task_id': 7, 'initial_state_index': 21, 'condition': 'hinglish', 'instruction': 'Stove chalu karo.', 'success': True, 'clip': 'media/layout/hinglish_t07_s21.mp4', 'poster': 'media/layout/hinglish_t07_s21_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task07_state21.mp4'}, {'task_id': 9, 'initial_state_index': 21, 'condition': 'hinglish', 'instruction': 'Wine bottle ko wine rack par rakho.', 'success': True, 'clip': 'media/layout/hinglish_t09_s21.mp4', 'poster': 'media/layout/hinglish_t09_s21_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task09_state21.mp4'}, {'task_id': 1, 'initial_state_index': 22, 'condition': 'hinglish', 'instruction': 'Bowl ko stove par rakho.', 'success': True, 'clip': 'media/layout/hinglish_t01_s22.mp4', 'poster': 'media/layout/hinglish_t01_s22_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task01_state22.mp4'}, {'task_id': 3, 'initial_state_index': 22, 'condition': 'hinglish', 'instruction': 'Cabinet ka sabse upar wala drawer kholo aur bowl ko uske andar rakho.', 'success': False, 'clip': 'media/layout/hinglish_t03_s22.mp4', 'poster': 'media/layout/hinglish_t03_s22_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task03_state22.mp4'}, {'task_id': 5, 'initial_state_index': 22, 'condition': 'hinglish', 'instruction': 'Plate ko stove ke saamne dhakelo.', 'success': False, 'clip': 'media/layout/hinglish_t05_s22.mp4', 'poster': 'media/layout/hinglish_t05_s22_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task05_state22.mp4'}, {'task_id': 7, 'initial_state_index': 22, 'condition': 'hinglish', 'instruction': 'Stove chalu karo.', 'success': True, 'clip': 'media/layout/hinglish_t07_s22.mp4', 'poster': 'media/layout/hinglish_t07_s22_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task07_state22.mp4'}, {'task_id': 9, 'initial_state_index': 22, 'condition': 'hinglish', 'instruction': 'Wine bottle ko wine rack par rakho.', 'success': True, 'clip': 'media/layout/hinglish_t09_s22.mp4', 'poster': 'media/layout/hinglish_t09_s22_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task09_state22.mp4'}, {'task_id': 1, 'initial_state_index': 23, 'condition': 'hinglish', 'instruction': 'Bowl ko stove par rakho.', 'success': True, 'clip': 'media/layout/hinglish_t01_s23.mp4', 'poster': 'media/layout/hinglish_t01_s23_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task01_state23.mp4'}, {'task_id': 3, 'initial_state_index': 23, 'condition': 'hinglish', 'instruction': 'Cabinet ka sabse upar wala drawer kholo aur bowl ko uske andar rakho.', 'success': False, 'clip': 'media/layout/hinglish_t03_s23.mp4', 'poster': 'media/layout/hinglish_t03_s23_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task03_state23.mp4'}, {'task_id': 5, 'initial_state_index': 23, 'condition': 'hinglish', 'instruction': 'Plate ko stove ke saamne dhakelo.', 'success': False, 'clip': 'media/layout/hinglish_t05_s23.mp4', 'poster': 'media/layout/hinglish_t05_s23_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task05_state23.mp4'}, {'task_id': 7, 'initial_state_index': 23, 'condition': 'hinglish', 'instruction': 'Stove chalu karo.', 'success': True, 'clip': 'media/layout/hinglish_t07_s23.mp4', 'poster': 'media/layout/hinglish_t07_s23_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task07_state23.mp4'}, {'task_id': 9, 'initial_state_index': 23, 'condition': 'hinglish', 'instruction': 'Wine bottle ko wine rack par rakho.', 'success': True, 'clip': 'media/layout/hinglish_t09_s23.mp4', 'poster': 'media/layout/hinglish_t09_s23_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task09_state23.mp4'}, {'task_id': 1, 'initial_state_index': 24, 'condition': 'hinglish', 'instruction': 'Bowl ko stove par rakho.', 'success': True, 'clip': 'media/layout/hinglish_t01_s24.mp4', 'poster': 'media/layout/hinglish_t01_s24_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task01_state24.mp4'}, {'task_id': 3, 'initial_state_index': 24, 'condition': 'hinglish', 'instruction': 'Cabinet ka sabse upar wala drawer kholo aur bowl ko uske andar rakho.', 'success': False, 'clip': 'media/layout/hinglish_t03_s24.mp4', 'poster': 'media/layout/hinglish_t03_s24_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task03_state24.mp4'}, {'task_id': 5, 'initial_state_index': 24, 'condition': 'hinglish', 'instruction': 'Plate ko stove ke saamne dhakelo.', 'success': False, 'clip': 'media/layout/hinglish_t05_s24.mp4', 'poster': 'media/layout/hinglish_t05_s24_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task05_state24.mp4'}, {'task_id': 7, 'initial_state_index': 24, 'condition': 'hinglish', 'instruction': 'Stove chalu karo.', 'success': True, 'clip': 'media/layout/hinglish_t07_s24.mp4', 'poster': 'media/layout/hinglish_t07_s24_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task07_state24.mp4'}, {'task_id': 9, 'initial_state_index': 24, 'condition': 'hinglish', 'instruction': 'Wine bottle ko wine rack par rakho.', 'success': False, 'clip': 'media/layout/hinglish_t09_s24.mp4', 'poster': 'media/layout/hinglish_t09_s24_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task09_state24.mp4'}] | |
| INDIC_SOURCE={'repository': 'krutrim-ai-labs/IndicVisionBench', 'revision': '45269760b15938c6a345f74373f56c7a5ab7ecc8', 'ground_truth': 'Published MCQ reference letters; not model-generated or LLM-judged', 'matched_items_per_language': 98} | |
| EXAMPLE_SELECTION=[{'id': 'vqa_par_0001', 'pattern': '111'}, {'id': 'vqa_par_0365', 'pattern': '100'}, {'id': 'vqa_par_0432', 'pattern': '100'}, {'id': 'vqa_par_0103', 'pattern': '110'}, {'id': 'vqa_par_0354', 'pattern': '101'}, {'id': 'vqa_par_0852', 'pattern': '011'}, {'id': 'vqa_par_0979', 'pattern': '010'}, {'id': 'vqa_par_0250', 'pattern': '000'}] | |
| def load(): | |
| ROOT=Path(__file__).resolve().parents[1] | |
| def read(relative):return json.loads((ROOT/relative).read_text(encoding='utf-8')) | |
| LANGUAGES=['english','hindi','hinglish'] | |
| corpus=read('data/backbone/corpus.json');items={r['id']:r for r in corpus} | |
| raw=[json.loads(line) for line in (ROOT/'data/backbone/predictions.jsonl').read_text(encoding='utf-8').splitlines()] | |
| assert len(corpus)==158 and len(raw)==474 | |
| keys={(r['id'],r['language']) for r in raw} | |
| assert len(keys)==len(raw) and keys=={(r['id'],l) for r in corpus for l in r['variants']} | |
| suffix={'english':'Answer with only the correct option letter A, B, C, or D.', | |
| 'hindi':'केवल सही विकल्प का अक्षर A, B, C या D लिखें।', | |
| 'hinglish':'Sirf sahi option ka letter A, B, C ya D likho.'} | |
| for r in raw: | |
| item=items[r['id']];variant=item['variants'][r['language']] | |
| expected=variant['question']+'\n'+'\n'.join(f'{letter}. {option}' for letter,option in zip('ABCD',variant['options']))+'\n'+suffix[r['language']] | |
| assert r['prompt']==expected and r['answer']==item['answer'] | |
| r['computed_correct']=parse_choice(r['response'])==item['answer'] | |
| assert r['computed_correct']==r['correct'] | |
| predictions=pd.DataFrame(raw) | |
| def summary(frame,groups): | |
| result=frame.groupby(groups,sort=False).computed_correct.agg(correct='sum',total='size').reset_index() | |
| result['accuracy_pct']=100*result.correct/result.total | |
| return result | |
| indic=predictions[predictions.dataset=='IndicVisionBench_parallel'] | |
| indic_results=summary(indic,['language']).set_index('language').loc[LANGUAGES].reset_index() | |
| assert indic_results.correct.tolist()==[92,88,88] | |
| paired_vqa=paired_items(indic.to_dict('records')) | |
| contingencies=[dict(left=LABELS[a],right=LABELS[b],**contingency([r[a]['correct'] for r in paired_vqa.values()],[r[b]['correct'] for r in paired_vqa.values()])) for a,b in [('hindi','hinglish'),('english','hindi'),('english','hinglish')]] | |
| contingency_table=pd.DataFrame(contingencies) | |
| selection={'examples':EXAMPLE_SELECTION};indic_source=INDIC_SOURCE | |
| tasks=pd.read_csv(ROOT/'data/translations/tasks.csv',keep_default_na=False) | |
| translations=pd.read_csv(ROOT/'data/translations/translations.csv',keep_default_na=False) | |
| review=pd.read_csv(ROOT/'data/translations/semantic_review.csv',keep_default_na=False) | |
| semantic=translations.merge(review,on=['service','language','task_id'],validate='one_to_one') | |
| assert len(semantic)==60 | |
| episodes=read('data/vla/episodes.json');matrices=validate_pairs(episodes) | |
| assert [int(matrices[l].sum()) for l in LANGUAGES]==[195,58,113] | |
| baseline=read('data/vla/english_baseline_500.json') | |
| assert len(baseline)==500 and sum(r['success'] for r in baseline)==490 | |
| controls={(r['task_id'],r['initial_state_index']):r for r in baseline if 20<=r['initial_state_index']<=39} | |
| for r in episodes: | |
| if r['condition']=='english': | |
| old=controls[r['task_id'],r['initial_state_index']] | |
| assert r['success']==old['success'] and r['instruction']==old['instruction'] | |
| overall=pd.DataFrame([dict(language=l,successes=int(matrices[l].sum()),episodes=200,success_pct=100*matrices[l].mean(),ci_low=100*wilson(int(matrices[l].sum()),200)[0],ci_high=100*wilson(int(matrices[l].sum()),200)[1]) for l in LANGUAGES]) | |
| stored=pd.read_csv(ROOT/'data/vla/paired_differences.csv') | |
| recorded={'paired_differences':{}} | |
| for (a,b),(_,r) in zip([('hindi','english'),('hinglish','english'),('hinglish','hindi')],stored.iterrows()): | |
| recorded['paired_differences'][a+'_minus_'+b]=dict(difference=r.difference_pp/100,ci95=[r.ci_low_pp/100,r.ci_high_pp/100],**{k:int(r[k]) for k in ['both_success','left_only','right_only','both_failure']}) | |
| media=DISPLAY_CASES;matrix_media=MATRIX_MEDIA | |
| french=pd.DataFrame(read('data/french/selected_trials.json')) | |
| assert len(french)==38 and set(french.task_id)=={0,3,4,5} | |
| structure_frame=pd.DataFrame(read('data/instruction_structure/episodes.json')) | |
| counts=structure_frame.groupby('structure').success.agg(['sum','size']) | |
| assert [tuple(counts.loc[k]) for k in ['Keywords only','Normal target-first','Negation-first / reordered']]==[(2,4),(7,8),(4,8)] | |
| return dict(locals(),paired_difference=paired_difference,wilson=wilson) | |
Xet Storage Details
- Size:
- 18.7 kB
- Xet hash:
- e5bcceb5a324e7c07bf288f1b71c80815b87c1815edd3aba91c8411e3bea80f7
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.