SolusOps/fatima / notebook_backend /editorial_records.py
SolusOps's picture
download
raw
18.7 kB
"""Load only the curated public evidence. No model, simulator or research archive access."""
from pathlib import Path
import json
import numpy as np
import pandas as pd
from analysis import parse_choice,validate_pairs,paired_difference,wilson,paired_items,contingency
from presentation import LABELS
DISPLAY_CASES=[{'id': 'case_matched_t04_s20_english', 'group': 'matched_language_contrast', 'task_id': 4, 'initial_state_index': 20, 'instruction': 'put the bowl on top of the cabinet', 'clip': 'media/layout/case_matched_t04_s20_english.mp4', 'poster': 'media/layout/case_matched_t04_s20_english_final_poster.png', 'success': True, 'language': 'english', 'full_recording': 'media/layout/full/case_matched_t04_s20_english.mp4'}, {'id': 'case_matched_t04_s20_hindi', 'group': 'matched_language_contrast', 'task_id': 4, 'initial_state_index': 20, 'instruction': 'बाउल को कैबिनेट के ऊपर रखो।', 'clip': 'media/layout/case_matched_t04_s20_hindi.mp4', 'poster': 'media/layout/case_matched_t04_s20_hindi_final_poster.png', 'success': False, 'language': 'hindi', 'full_recording': 'media/layout/full/case_matched_t04_s20_hindi.mp4'}, {'id': 'case_matched_t04_s20_hinglish', 'group': 'matched_language_contrast', 'task_id': 4, 'initial_state_index': 20, 'instruction': 'Bowl ko cabinet ke upar rakho.', 'clip': 'media/layout/case_matched_t04_s20_hinglish.mp4', 'poster': 'media/layout/case_matched_t04_s20_hinglish_final_poster.png', 'success': True, 'language': 'hinglish', 'full_recording': 'media/layout/full/case_matched_t04_s20_hinglish.mp4'}, {'id': 'case_hindi_t08_s20_success', 'group': 'ordinary_success', 'task_id': 8, 'initial_state_index': 20, 'instruction': 'बाउल को प्लेट पर रखो।', 'clip': 'media/layout/case_hindi_t08_s20_success.mp4', 'poster': 'media/layout/case_hindi_t08_s20_success_final_poster.png', 'success': True, 'language': 'hindi', 'full_recording': 'media/layout/full/case_hindi_t08_s20_success.mp4'}, {'id': 'case_unusual_hindi_t06_s01', 'group': 'unusual_success', 'task_id': 6, 'initial_state_index': 1, 'instruction': 'क्रीम चीज़ को बाउल में रखो।', 'clip': 'media/layout/case_unusual_hindi_t06_s01.mp4', 'poster': 'media/layout/case_unusual_hindi_t06_s01_final_poster.png', 'success': True, 'language': 'hindi', 'full_recording': 'media/layout/full/case_unusual_hindi_t06_s01.mp4'}, {'id': 'case_greeting_english_t02_s00', 'group': 'recurring_bowl_to_stove', 'task_id': 2, 'initial_state_index': 0, 'instruction': 'Hello.', 'clip': 'media/layout/case_greeting_english_t02_s00.mp4', 'poster': 'media/layout/case_greeting_english_t02_s00_final_poster.png', 'success': None, 'language': 'english', 'full_recording': 'media/layout/full/case_greeting_english_t02_s00.mp4'}, {'id': 'case_french_cabinet_baseline', 'group': 'french_illustrative_contrast', 'task_id': 4, 'initial_state_index': 0, 'instruction': "Placez le bol sur le dessus de l'armoire.", 'clip': 'media/layout/case_french_cabinet_baseline.mp4', 'poster': 'media/layout/case_french_cabinet_baseline_final_poster.png', 'success': False, 'language': 'french', 'full_recording': 'media/layout/full/case_french_cabinet_baseline.mp4'}, {'id': 'french_selection_28', 'group': 'french_lexical', 'task_id': 0, 'initial_state_index': 0, 'instruction': "Ouvrez le tiroir du milieu de l'armoire.", 'clip': 'media/layout/french_selection_28.mp4', 'poster': 'media/layout/french_selection_28_final_poster.png', 'success': False, 'language': 'french', 'condition': 'A_fully_french', 'full_recording': 'media/layout/full/french_selection_28.mp4', 'run_id': 'middle_drawer_codeswitch'}, {'id': 'french_selection_29', 'group': 'french_lexical', 'task_id': 0, 'initial_state_index': 0, 'instruction': "Ouvrez the middle drawer de l'armoire.", 'clip': 'media/layout/french_selection_29.mp4', 'poster': 'media/layout/french_selection_29_final_poster.png', 'success': True, 'language': 'french', 'condition': 'B_english_object_only', 'full_recording': 'media/layout/full/french_selection_29.mp4', 'run_id': 'middle_drawer_codeswitch'}, {'id': 'french_selection_31', 'group': 'french_lexical', 'task_id': 5, 'initial_state_index': 0, 'instruction': "Push l'assiette vers l'avant de la cuisinière.", 'clip': 'media/layout/french_selection_31.mp4', 'poster': 'media/layout/french_selection_31_final_poster.png', 'success': False, 'language': 'french', 'condition': 'A_verb_only', 'full_recording': 'media/layout/full/french_selection_31.mp4', 'run_id': 'plate_stove_ADGH_stop_first_success'}, {'id': 'french_selection_32', 'group': 'french_lexical', 'task_id': 5, 'initial_state_index': 0, 'instruction': "Push the plate vers l'avant de la cuisinière.", 'clip': 'media/layout/french_selection_32.mp4', 'poster': 'media/layout/french_selection_32_final_poster.png', 'success': True, 'language': 'french', 'condition': 'D_verb_object', 'full_recording': 'media/layout/full/french_selection_32.mp4', 'run_id': 'plate_stove_ADGH_stop_first_success'}, {'id': 'french_selection_44', 'group': 'french_lexical', 'task_id': 3, 'initial_state_index': 0, 'instruction': "Ouvrez the top drawer et placez le bol à l'intérieur.", 'clip': 'media/layout/french_selection_44.mp4', 'poster': 'media/layout/french_selection_44_final_poster.png', 'success': False, 'language': 'french', 'condition': 'C_english_destination_only', 'full_recording': 'media/layout/full/french_selection_44.mp4', 'run_id': 'top_drawer_bowl_conditions_3_4_five'}, {'id': 'french_selection_49', 'group': 'french_lexical', 'task_id': 3, 'initial_state_index': 0, 'instruction': "Ouvrez the top drawer et placez the bowl à l'intérieur.", 'clip': 'media/layout/french_selection_49.mp4', 'poster': 'media/layout/french_selection_49_final_poster.png', 'success': True, 'language': 'french', 'condition': 'D_english_object_and_destination', 'full_recording': 'media/layout/full/french_selection_49.mp4', 'run_id': 'top_drawer_bowl_conditions_3_4_five'}]
MATRIX_MEDIA=[{'task_id': 1, 'initial_state_index': 20, 'condition': 'hinglish', 'instruction': 'Bowl ko stove par rakho.', 'success': True, 'clip': 'media/layout/hinglish_t01_s20.mp4', 'poster': 'media/layout/hinglish_t01_s20_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task01_state20.mp4'}, {'task_id': 3, 'initial_state_index': 20, 'condition': 'hinglish', 'instruction': 'Cabinet ka sabse upar wala drawer kholo aur bowl ko uske andar rakho.', 'success': False, 'clip': 'media/layout/hinglish_t03_s20.mp4', 'poster': 'media/layout/hinglish_t03_s20_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task03_state20.mp4'}, {'task_id': 5, 'initial_state_index': 20, 'condition': 'hinglish', 'instruction': 'Plate ko stove ke saamne dhakelo.', 'success': False, 'clip': 'media/layout/hinglish_t05_s20.mp4', 'poster': 'media/layout/hinglish_t05_s20_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task05_state20.mp4'}, {'task_id': 7, 'initial_state_index': 20, 'condition': 'hinglish', 'instruction': 'Stove chalu karo.', 'success': True, 'clip': 'media/layout/hinglish_t07_s20.mp4', 'poster': 'media/layout/hinglish_t07_s20_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task07_state20.mp4'}, {'task_id': 9, 'initial_state_index': 20, 'condition': 'hinglish', 'instruction': 'Wine bottle ko wine rack par rakho.', 'success': True, 'clip': 'media/layout/hinglish_t09_s20.mp4', 'poster': 'media/layout/hinglish_t09_s20_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task09_state20.mp4'}, {'task_id': 1, 'initial_state_index': 21, 'condition': 'hinglish', 'instruction': 'Bowl ko stove par rakho.', 'success': True, 'clip': 'media/layout/hinglish_t01_s21.mp4', 'poster': 'media/layout/hinglish_t01_s21_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task01_state21.mp4'}, {'task_id': 3, 'initial_state_index': 21, 'condition': 'hinglish', 'instruction': 'Cabinet ka sabse upar wala drawer kholo aur bowl ko uske andar rakho.', 'success': False, 'clip': 'media/layout/hinglish_t03_s21.mp4', 'poster': 'media/layout/hinglish_t03_s21_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task03_state21.mp4'}, {'task_id': 5, 'initial_state_index': 21, 'condition': 'hinglish', 'instruction': 'Plate ko stove ke saamne dhakelo.', 'success': False, 'clip': 'media/layout/hinglish_t05_s21.mp4', 'poster': 'media/layout/hinglish_t05_s21_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task05_state21.mp4'}, {'task_id': 7, 'initial_state_index': 21, 'condition': 'hinglish', 'instruction': 'Stove chalu karo.', 'success': True, 'clip': 'media/layout/hinglish_t07_s21.mp4', 'poster': 'media/layout/hinglish_t07_s21_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task07_state21.mp4'}, {'task_id': 9, 'initial_state_index': 21, 'condition': 'hinglish', 'instruction': 'Wine bottle ko wine rack par rakho.', 'success': True, 'clip': 'media/layout/hinglish_t09_s21.mp4', 'poster': 'media/layout/hinglish_t09_s21_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task09_state21.mp4'}, {'task_id': 1, 'initial_state_index': 22, 'condition': 'hinglish', 'instruction': 'Bowl ko stove par rakho.', 'success': True, 'clip': 'media/layout/hinglish_t01_s22.mp4', 'poster': 'media/layout/hinglish_t01_s22_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task01_state22.mp4'}, {'task_id': 3, 'initial_state_index': 22, 'condition': 'hinglish', 'instruction': 'Cabinet ka sabse upar wala drawer kholo aur bowl ko uske andar rakho.', 'success': False, 'clip': 'media/layout/hinglish_t03_s22.mp4', 'poster': 'media/layout/hinglish_t03_s22_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task03_state22.mp4'}, {'task_id': 5, 'initial_state_index': 22, 'condition': 'hinglish', 'instruction': 'Plate ko stove ke saamne dhakelo.', 'success': False, 'clip': 'media/layout/hinglish_t05_s22.mp4', 'poster': 'media/layout/hinglish_t05_s22_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task05_state22.mp4'}, {'task_id': 7, 'initial_state_index': 22, 'condition': 'hinglish', 'instruction': 'Stove chalu karo.', 'success': True, 'clip': 'media/layout/hinglish_t07_s22.mp4', 'poster': 'media/layout/hinglish_t07_s22_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task07_state22.mp4'}, {'task_id': 9, 'initial_state_index': 22, 'condition': 'hinglish', 'instruction': 'Wine bottle ko wine rack par rakho.', 'success': True, 'clip': 'media/layout/hinglish_t09_s22.mp4', 'poster': 'media/layout/hinglish_t09_s22_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task09_state22.mp4'}, {'task_id': 1, 'initial_state_index': 23, 'condition': 'hinglish', 'instruction': 'Bowl ko stove par rakho.', 'success': True, 'clip': 'media/layout/hinglish_t01_s23.mp4', 'poster': 'media/layout/hinglish_t01_s23_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task01_state23.mp4'}, {'task_id': 3, 'initial_state_index': 23, 'condition': 'hinglish', 'instruction': 'Cabinet ka sabse upar wala drawer kholo aur bowl ko uske andar rakho.', 'success': False, 'clip': 'media/layout/hinglish_t03_s23.mp4', 'poster': 'media/layout/hinglish_t03_s23_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task03_state23.mp4'}, {'task_id': 5, 'initial_state_index': 23, 'condition': 'hinglish', 'instruction': 'Plate ko stove ke saamne dhakelo.', 'success': False, 'clip': 'media/layout/hinglish_t05_s23.mp4', 'poster': 'media/layout/hinglish_t05_s23_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task05_state23.mp4'}, {'task_id': 7, 'initial_state_index': 23, 'condition': 'hinglish', 'instruction': 'Stove chalu karo.', 'success': True, 'clip': 'media/layout/hinglish_t07_s23.mp4', 'poster': 'media/layout/hinglish_t07_s23_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task07_state23.mp4'}, {'task_id': 9, 'initial_state_index': 23, 'condition': 'hinglish', 'instruction': 'Wine bottle ko wine rack par rakho.', 'success': True, 'clip': 'media/layout/hinglish_t09_s23.mp4', 'poster': 'media/layout/hinglish_t09_s23_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task09_state23.mp4'}, {'task_id': 1, 'initial_state_index': 24, 'condition': 'hinglish', 'instruction': 'Bowl ko stove par rakho.', 'success': True, 'clip': 'media/layout/hinglish_t01_s24.mp4', 'poster': 'media/layout/hinglish_t01_s24_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task01_state24.mp4'}, {'task_id': 3, 'initial_state_index': 24, 'condition': 'hinglish', 'instruction': 'Cabinet ka sabse upar wala drawer kholo aur bowl ko uske andar rakho.', 'success': False, 'clip': 'media/layout/hinglish_t03_s24.mp4', 'poster': 'media/layout/hinglish_t03_s24_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task03_state24.mp4'}, {'task_id': 5, 'initial_state_index': 24, 'condition': 'hinglish', 'instruction': 'Plate ko stove ke saamne dhakelo.', 'success': False, 'clip': 'media/layout/hinglish_t05_s24.mp4', 'poster': 'media/layout/hinglish_t05_s24_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task05_state24.mp4'}, {'task_id': 7, 'initial_state_index': 24, 'condition': 'hinglish', 'instruction': 'Stove chalu karo.', 'success': True, 'clip': 'media/layout/hinglish_t07_s24.mp4', 'poster': 'media/layout/hinglish_t07_s24_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task07_state24.mp4'}, {'task_id': 9, 'initial_state_index': 24, 'condition': 'hinglish', 'instruction': 'Wine bottle ko wine rack par rakho.', 'success': False, 'clip': 'media/layout/hinglish_t09_s24.mp4', 'poster': 'media/layout/hinglish_t09_s24_final_poster.png', 'full_recording': 'media/rollouts/hinglish_task09_state24.mp4'}]
INDIC_SOURCE={'repository': 'krutrim-ai-labs/IndicVisionBench', 'revision': '45269760b15938c6a345f74373f56c7a5ab7ecc8', 'ground_truth': 'Published MCQ reference letters; not model-generated or LLM-judged', 'matched_items_per_language': 98}
EXAMPLE_SELECTION=[{'id': 'vqa_par_0001', 'pattern': '111'}, {'id': 'vqa_par_0365', 'pattern': '100'}, {'id': 'vqa_par_0432', 'pattern': '100'}, {'id': 'vqa_par_0103', 'pattern': '110'}, {'id': 'vqa_par_0354', 'pattern': '101'}, {'id': 'vqa_par_0852', 'pattern': '011'}, {'id': 'vqa_par_0979', 'pattern': '010'}, {'id': 'vqa_par_0250', 'pattern': '000'}]
def load():
ROOT=Path(__file__).resolve().parents[1]
def read(relative):return json.loads((ROOT/relative).read_text(encoding='utf-8'))
LANGUAGES=['english','hindi','hinglish']
corpus=read('data/backbone/corpus.json');items={r['id']:r for r in corpus}
raw=[json.loads(line) for line in (ROOT/'data/backbone/predictions.jsonl').read_text(encoding='utf-8').splitlines()]
assert len(corpus)==158 and len(raw)==474
keys={(r['id'],r['language']) for r in raw}
assert len(keys)==len(raw) and keys=={(r['id'],l) for r in corpus for l in r['variants']}
suffix={'english':'Answer with only the correct option letter A, B, C, or D.',
'hindi':'केवल सही विकल्प का अक्षर A, B, C या D लिखें।',
'hinglish':'Sirf sahi option ka letter A, B, C ya D likho.'}
for r in raw:
item=items[r['id']];variant=item['variants'][r['language']]
expected=variant['question']+'\n'+'\n'.join(f'{letter}. {option}' for letter,option in zip('ABCD',variant['options']))+'\n'+suffix[r['language']]
assert r['prompt']==expected and r['answer']==item['answer']
r['computed_correct']=parse_choice(r['response'])==item['answer']
assert r['computed_correct']==r['correct']
predictions=pd.DataFrame(raw)
def summary(frame,groups):
result=frame.groupby(groups,sort=False).computed_correct.agg(correct='sum',total='size').reset_index()
result['accuracy_pct']=100*result.correct/result.total
return result
indic=predictions[predictions.dataset=='IndicVisionBench_parallel']
indic_results=summary(indic,['language']).set_index('language').loc[LANGUAGES].reset_index()
assert indic_results.correct.tolist()==[92,88,88]
paired_vqa=paired_items(indic.to_dict('records'))
contingencies=[dict(left=LABELS[a],right=LABELS[b],**contingency([r[a]['correct'] for r in paired_vqa.values()],[r[b]['correct'] for r in paired_vqa.values()])) for a,b in [('hindi','hinglish'),('english','hindi'),('english','hinglish')]]
contingency_table=pd.DataFrame(contingencies)
selection={'examples':EXAMPLE_SELECTION};indic_source=INDIC_SOURCE
tasks=pd.read_csv(ROOT/'data/translations/tasks.csv',keep_default_na=False)
translations=pd.read_csv(ROOT/'data/translations/translations.csv',keep_default_na=False)
review=pd.read_csv(ROOT/'data/translations/semantic_review.csv',keep_default_na=False)
semantic=translations.merge(review,on=['service','language','task_id'],validate='one_to_one')
assert len(semantic)==60
episodes=read('data/vla/episodes.json');matrices=validate_pairs(episodes)
assert [int(matrices[l].sum()) for l in LANGUAGES]==[195,58,113]
baseline=read('data/vla/english_baseline_500.json')
assert len(baseline)==500 and sum(r['success'] for r in baseline)==490
controls={(r['task_id'],r['initial_state_index']):r for r in baseline if 20<=r['initial_state_index']<=39}
for r in episodes:
if r['condition']=='english':
old=controls[r['task_id'],r['initial_state_index']]
assert r['success']==old['success'] and r['instruction']==old['instruction']
overall=pd.DataFrame([dict(language=l,successes=int(matrices[l].sum()),episodes=200,success_pct=100*matrices[l].mean(),ci_low=100*wilson(int(matrices[l].sum()),200)[0],ci_high=100*wilson(int(matrices[l].sum()),200)[1]) for l in LANGUAGES])
stored=pd.read_csv(ROOT/'data/vla/paired_differences.csv')
recorded={'paired_differences':{}}
for (a,b),(_,r) in zip([('hindi','english'),('hinglish','english'),('hinglish','hindi')],stored.iterrows()):
recorded['paired_differences'][a+'_minus_'+b]=dict(difference=r.difference_pp/100,ci95=[r.ci_low_pp/100,r.ci_high_pp/100],**{k:int(r[k]) for k in ['both_success','left_only','right_only','both_failure']})
media=DISPLAY_CASES;matrix_media=MATRIX_MEDIA
french=pd.DataFrame(read('data/french/selected_trials.json'))
assert len(french)==38 and set(french.task_id)=={0,3,4,5}
structure_frame=pd.DataFrame(read('data/instruction_structure/episodes.json'))
counts=structure_frame.groupby('structure').success.agg(['sum','size'])
assert [tuple(counts.loc[k]) for k in ['Keywords only','Normal target-first','Negation-first / reordered']]==[(2,4),(7,8),(4,8)]
return dict(locals(),paired_difference=paired_difference,wilson=wilson)

Xet Storage Details

Size:
18.7 kB
·
Xet hash:
e5bcceb5a324e7c07bf288f1b71c80815b87c1815edd3aba91c8411e3bea80f7

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.