recovery related files

more recovery related files

recovery/MIDDLE_E4B_RETRY32_COLDSTART.py

@@ -0,0 +1,416 @@
+# =============================================================
+# SINHALAMMLU — MIDDLE E4B V2 TARGETED RECOVERY, 32 QUESTIONS
+# Self-contained fresh Kaggle session, with T4 x2 and Internet ON.
+# Attach: (1) original 41 JSON test question files (questions-set)
+#         (2) E4B_COMBINED_2206_WITH_FAILED32.zip as a Kaggle dataset.
+# Saves independent per-question retry records, recovery ZIP and
+# FINAL 2238-row predictions.zip only after 32/32 valid recoveries.
+# NEVER trains on test data, NEVER guesses missing answers.
+# =============================================================
+import os, re, gc, json, time, hashlib, zipfile
+from pathlib import Path
+from datetime import datetime, timezone
+from collections import Counter
+
+import torch
+from huggingface_hub import snapshot_download
+from accelerate import init_empty_weights
+from transformers import AutoConfig, AutoProcessor, AutoModelForMultimodalLM
+
+ROOT = Path('/kaggle/working/SinhalaMMLU')
+RUN = ROOT / 'experiments/MIDDLE_E4B_RETRY32'
+STORE = RUN / 'retry_records'
+STORE.mkdir(parents=True, exist_ok=True)
+ARCHIVE = ROOT / 'archives/MIDDLE_E4B_RETRY32_LATEST.zip'
+ARCHIVE.parent.mkdir(parents=True, exist_ok=True)
+FINAL_ZIP = ROOT / 'archives/E4B_V2_FINAL_2238_predictions.zip'
+
+MODEL_REPO = 'google/gemma-4-E4B-it'
+REVISION = 'ee0ef6023621cff504d758262d4e04895a5af4a2'
+WEIGHT_SHA = 'cfbd3d2f1cd71bd471c37fe2bf8546d5028d41e5736f64e1ca6c6b8893125503'
+PROMPT_SHA = 'f53159bafe22a5d0063a065b6d3aef8167e7a6d6b998ce65e224feeeca68287d'
+V2_PROMPT = (
+    'Answer this Sinhala multiple-choice question. '
+    'Use your reasoning to select the correct option. '
+    'In your final response, provide a brief explanation '
+    'in English, approximately 90 words. '
+    'Then put ANSWER: X on the last line, '
+    'where X is A, B, C, or D. '
+    'Do not change the selected answer.'
+)
+assert hashlib.sha256(V2_PROMPT.encode('utf-8')).hexdigest() == PROMPT_SHA
+LETTER_TO_NUMBER = {'A':'1','B':'2','C':'3','D':'4'}
+FIELDS = ['q_no','subject','category','level','answer']
+TOKEN_BUDGET = 4096  # targeted single-question recovery only
+MAX_WALL_SECONDS = 4 * 3600
+
+def sha_file(path):
+    h = hashlib.sha256()
+    with open(path,'rb') as f:
+        for chunk in iter(lambda:f.read(8*1024*1024), b''):
+            h.update(chunk)
+    return h.hexdigest()
+
+def atomic_json(path,obj):
+    path.parent.mkdir(parents=True,exist_ok=True)
+    tmp = path.with_name(path.name+'.tmp')
+    tmp.write_text(json.dumps(obj,ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
+    os.replace(tmp,path)
+
+def now():
+    return datetime.now(timezone.utc).isoformat()
+
+def fingerprint(q):
+    keys=('source_file','source_position','q_no','subject','category','level','question','choices')
+    s=json.dumps({k:q[k] for k in keys},ensure_ascii=False,sort_keys=True,separators=(',',':'))
+    return hashlib.sha256(s.encode('utf-8')).hexdigest()
+
+print('▶ RETRY32-A — LOAD AUTHORITY PACKAGE AND TEST DATA',flush=True)
+# Kaggle may mount the ZIP intact OR automatically extract it.
+# Both layouts are supported, with identical content fingerprints.
+required_names=['E4B_COMBINED_VALID_2206_INDEXED.json','E4B_FAILED_ATTEMPTS_32.json',
+                'E4B_DATASET_SHA256_41.json','E4B_MERGE_AUDIT.json']
+packages=[]
+for p in Path('/kaggle/input').rglob('*.zip'):
+    try:
+        with zipfile.ZipFile(p) as z:
+            if set(required_names) <= set(z.namelist()):
+                packages.append(('zip',p))
+    except (zipfile.BadZipFile,OSError):
+        pass
+for p in Path('/kaggle/input').rglob(required_names[0]):
+    if all((p.parent/n).is_file() for n in required_names):
+        packages.append(('extracted',p.parent))
+# An uploaded ZIP may be visible both ways; prefer its intact version.
+zipped=[p for p in packages if p[0]=='zip']
+if zipped:packages=zipped
+assert len(packages)==1, f'Attach the combined-2206 package exactly once. Found: {packages}'
+package_kind,PACKAGE=packages[0]
+if package_kind=='zip':
+    with zipfile.ZipFile(PACKAGE) as z:
+        assert z.testzip() is None
+        source_bytes={name:z.read(name) for name in required_names}
+else:
+    source_bytes={name:(PACKAGE/name).read_bytes() for name in required_names}
+previous=json.loads(source_bytes[required_names[0]])
+failures=json.loads(source_bytes[required_names[1]])
+files_hashes=json.loads(source_bytes[required_names[2]])
+merge_audit=json.loads(source_bytes[required_names[3]])
+package_content_sha=hashlib.sha256(b''.join(
+    name.encode('utf-8') + b'\0' + source_bytes[name]
+    for name in required_names)).hexdigest()
+assert merge_audit['prompt_sha256']==PROMPT_SHA
+assert merge_audit['revision']==REVISION
+assert merge_audit['valid']==2206 and merge_audit['failed']==32
+assert len(previous)==2206 and len(failures)==32 and len(files_hashes)==41
+known_valid={int(r['dataset_index']):r for r in previous}
+fail_by_idx={int(r['dataset_index']):r for r in failures}
+assert len(known_valid)==2206 and len(fail_by_idx)==32
+assert not (known_valid.keys() & fail_by_idx.keys())
+assert set(known_valid)|set(fail_by_idx)==set(range(2238))
+
+question_dirs=[]
+for p in Path('/kaggle/input').rglob('Arts_Humanities_easy.json'):
+    d=p.parent
+    if len(list(d.glob('*.json')))!=41:continue
+    if all((d/f['filename']).is_file() and sha_file(d/f['filename'])==f['sha256']
+           for f in files_hashes):
+        question_dirs.append(d)
+assert len(question_dirs)==1,f'Need original unchanged 41-file questions-set: {question_dirs}'
+TEST_DIR=question_dirs[0]
+questions=[]
+for path in sorted(TEST_DIR.glob('*.json'),key=lambda p:p.name.casefold()):
+    obj=json.loads(path.read_text(encoding='utf-8'))
+    if isinstance(obj,dict):
+        obj=next((obj[k] for k in ('questions','data','items') if isinstance(obj.get(k),list)),None)
+    assert isinstance(obj,list),path
+    for pos,q in enumerate(obj):
+        assert len(q['choices'])==4
+        level=q['metadata']['difficulty']
+        assert level in ('easy','medium','hard')
+        questions.append({'source_file':path.name,'source_position':pos,'q_no':q['q_no'],
+                          'subject':q['subject'],'category':q['category'],'level':level,
+                          'question':q['question'],'choices':q['choices']})
+priority={'easy':0,'medium':1,'hard':2}
+questions.sort(key=lambda q:(priority[q['level']],q['source_file'].casefold(),q['source_position']))
+assert len(questions)==2238
+for i,q in enumerate(questions):q['dataset_index']=i
+# Strictly reconcile every question's provided metadata with saved results.
+for i in range(2238):
+    q=questions[i]
+    meta=known_valid[i] if i in known_valid else fail_by_idx[i]
+    assert all(q[k]==meta[k] for k in ('q_no','subject','category','level')),(i,meta)
+    if i in fail_by_idx:
+        f=fail_by_idx[i]
+        assert (q['source_file'],q['source_position'])==(f['source_file'],f['source_position'])
+        if f.get('input_fingerprint'):
+            assert fingerprint(q)==f['input_fingerprint'],i
+    else:
+        assert known_valid[i]['answer'] in '1234'
+indices=sorted(fail_by_idx)
+print('Package:',PACKAGE)
+print('Dataset:',TEST_DIR)
+print('Previously valid:',len(known_valid),'| failed:',len(indices))
+print('Retry indices:',indices,flush=True)
+
+# Immutable retry configuration. Per-run time limit is NOT part of the experiment identity.
+config={'experiment':'MIDDLE_E4B_RETRY32','model':MODEL_REPO,'revision':REVISION,
+        'prompt_sha256':PROMPT_SHA,'native_thinking':True,'batch_size':1,
+        'max_new_tokens':TOKEN_BUDGET,'do_sample':False,
+        'original_valid':2206,'recovery_indices':indices,
+        'dataset_sha256':dict((x['filename'],x['sha256']) for x in files_hashes),
+        'package_content_sha256':package_content_sha}
+cfg_path=RUN/'retry_config.json'
+if cfg_path.exists():
+    saved_cfg=json.loads(cfg_path.read_text('utf-8'))
+    assert saved_cfg==config,'Existing retry configuration conflicts with this package'
+else:
+    atomic_json(cfg_path,config)
+
+# Restore earlier retry records from a prior archive, if provided.
+# Never overwrite a valid local record with an older ZIP.
+prefix='experiments/MIDDLE_E4B_RETRY32/retry_records/'
+attached=[]
+for p in Path('/kaggle/input').rglob('*.zip'):
+    if p==PACKAGE:continue
+    try:
+        with zipfile.ZipFile(p) as z:
+            if any(n.startswith(prefix) for n in z.namelist()):attached.append(p)
+    except (OSError,zipfile.BadZipFile):pass
+if ARCHIVE.exists():attached.append(ARCHIVE)
+for p in attached:
+    with zipfile.ZipFile(p) as z:
+        assert z.testzip() is None
+        for name in z.namelist():
+            if not(name.startswith(prefix) and name.endswith('.json')):continue
+            filename=name[len(prefix):]
+            assert filename.startswith('question_') and '/' not in filename and '..' not in filename
+            dest=STORE/filename
+            raw=z.read(name)
+            if dest.exists():
+                assert dest.read_bytes()==raw,f'Conflicting retry record {dest}'
+            else:dest.write_bytes(raw)
+    print('Restored prior retry ZIP:',p,flush=True)
+
+def retry_path(i):return STORE/f'question_{i:04d}.json'
+
+def validate_retry(r):
+    i=int(r['dataset_index'])
+    assert i in fail_by_idx
+    q=questions[i]
+    assert r['input_fingerprint']==fingerprint(q),i
+    assert r['max_new_tokens']==TOKEN_BUDGET
+    assert r['prompt_sha256']==PROMPT_SHA
+    assert r['original_status']==fail_by_idx[i]['status']
+    sub=r['submission_record']
+    assert list(sub)==FIELDS
+    assert all(sub[k]==q[k] for k in FIELDS[:4])
+    if r['valid']:
+        assert r['status']=='completed'
+        assert r['prediction'] in LETTER_TO_NUMBER
+        assert sub['answer']==LETTER_TO_NUMBER[r['prediction']]
+        assert r['turn_end'] is True and not r['limit_hit']
+    else:
+        assert sub['answer'] is None
+
+def load_retries():
+    saved={}
+    for p in STORE.glob('question_*.json'):
+        r=json.loads(p.read_text(encoding='utf-8'))
+        validate_retry(r)
+        i=int(r['dataset_index'])
+        assert i not in saved
+        saved[i]=r
+    return saved
+
+def checkpoint():
+    saved=load_retries()
+    state={
+        'experiment':config['experiment'],'original_valid':2206,'retry_target':32,
+        'attempted':len(saved),'valid':sum(r['valid'] for r in saved.values()),
+        'invalid':sum(not r['valid'] for r in saved.values()),
+        'not_yet_retried':[i for i in indices if i not in saved],
+        'still_invalid':[i for i,r in sorted(saved.items()) if not r['valid']],
+        'status_counts':dict(Counter(r['status'] for r in saved.values())),
+        'status':'COMPLETE' if len(saved)==32 and all(r['valid'] for r in saved.values()) else 'IN_PROGRESS',
+        'timestamp_utc':now()}
+    atomic_json(RUN/'retry_manifest.json',state)
+    temp=ARCHIVE.with_name(ARCHIVE.name+'.tmp')
+    with zipfile.ZipFile(temp,'w',zipfile.ZIP_DEFLATED,compresslevel=5) as z:
+        for p in sorted(RUN.rglob('*')):
+            if p.is_file():z.write(p,str(p.relative_to(ROOT)))
+    os.replace(temp,ARCHIVE)
+    with zipfile.ZipFile(ARCHIVE) as z:
+        assert z.testzip() is None
+        assert sum(n.startswith(prefix) and n.endswith('.json') for n in z.namelist())==len(saved)
+    return state
+
+state=checkpoint()
+print('\n▶ RETRY32-B — INITIAL STATUS')
+print('Already retried:',state['attempted'],'| valid:',state['valid'],'| still to retry:',len(state['not_yet_retried']))
+
+# Do not load a 16GB model if there are no questions left to attempt.
+if state['not_yet_retried']:
+    print('\n▶ RETRY32-C — VERIFY AND LOAD GEMMA 4 E4B',flush=True)
+    assert torch.cuda.device_count()==2,'Enable T4 x2'
+    MODEL_DIR=ROOT/'weights/gemma4_e4b_it'
+    MODEL_DIR.mkdir(parents=True,exist_ok=True)
+    weights=MODEL_DIR/'model.safetensors'
+    if not weights.exists() or weights.stat().st_size!=15992595884:
+        snapshot_download(repo_id=MODEL_REPO,revision=REVISION,local_dir=str(MODEL_DIR),
+                          token=os.getenv('HF_TOKEN') or None)
+    assert weights.stat().st_size==15992595884
+    assert sha_file(weights)==WEIGHT_SHA
+    print('Pinned checkpoint SHA256 VERIFIED',flush=True)
+    if 'gemma_model' not in globals():
+        assert all(torch.cuda.memory_allocated(i)<1024**3 for i in range(2)),\
+               'GPU memory occupied; restart kernel rather than loading a second model'
+        mc=AutoConfig.from_pretrained(str(MODEL_DIR),local_files_only=True)
+        processor=AutoProcessor.from_pretrained(str(MODEL_DIR),local_files_only=True)
+        with init_empty_weights():
+            skeleton=AutoModelForMultimodalLM.from_config(mc,torch_dtype=torch.float16)
+        layer_prefix='model.language_model.layers'
+        assert len(dict(skeleton.named_modules())[layer_prefix])==42
+        decoder={f'{layer_prefix}.{i}' for i in range(42)}
+        dm={}
+        def includes_decoder(name):
+            p=name+'.' if name else ''
+            return any(d==name or d.startswith(p) for d in decoder)
+        def assign(module,name=''):
+            if name in decoder:
+                dm[name]='cuda:1';return
+            if name and not includes_decoder(name):
+                dm[name]='cuda:0';return
+            assert not list(module.named_parameters(recurse=False))
+            assert not list(module.named_buffers(recurse=False))
+            for c,child in module.named_children():
+                assign(child,f'{name}.{c}' if name else c)
+        assign(skeleton)
+        assert len(dm)==53 and '' not in dm
+        for n,_ in list(skeleton.named_parameters())+list(skeleton.named_buffers()):
+            assert sum(n==k or n.startswith(k+'.') for k in dm)==1,n
+        del skeleton;gc.collect()
+        gemma_model=AutoModelForMultimodalLM.from_pretrained(
+            str(MODEL_DIR),local_files_only=True,dtype=torch.float16,
+            attn_implementation='sdpa',device_map=dm,low_cpu_mem_usage=True)
+        gemma_model.eval()
+    else:
+        assert 'processor' in globals(),'Found model but not processor'
+    assert type(gemma_model).__name__=='Gemma4ForConditionalGeneration'
+    assert sum(p.numel() for p in gemma_model.parameters())==7941100832
+    assert not gemma_model.training
+    print('E4B loaded, GPU GiB:',[round(torch.cuda.memory_allocated(i)/2**30,3) for i in range(2)],flush=True)
+
+    tokenizer=processor.tokenizer
+    tokenizer.padding_side='left'
+    pad_id=tokenizer.pad_token_id
+    turn_id=tokenizer.convert_tokens_to_ids('<turn|>')
+    assert pad_id is not None and tokenizer.decode([turn_id],skip_special_tokens=False)=='<turn|>'
+    eos=gemma_model.generation_config.eos_token_id
+    stop_ids=({eos} if isinstance(eos,int) else set(eos or []))|{turn_id}
+
+    def score_single(q):
+        opts='\n'.join(f'{letter}. {choice}' for letter,choice in zip('ABCD',q['choices']))
+        text=V2_PROMPT+'\n\n'+q['question']+'\n\n'+opts
+        msg=[{'role':'user','content':[{'type':'text','text':text}]}]
+        enc=processor.apply_chat_template(msg,tokenize=True,add_generation_prompt=True,
+            enable_thinking=True,return_dict=True,return_tensors='pt').to('cuda:0')
+        width=enc['input_ids'].shape[1]
+        for gpu in range(2):torch.cuda.synchronize(gpu)
+        t=time.perf_counter()
+        with torch.inference_mode():
+            sequence=gemma_model.generate(**enc,max_new_tokens=TOKEN_BUDGET,
+                do_sample=False,use_cache=True,pad_token_id=pad_id)
+        for gpu in range(2):torch.cuda.synchronize(gpu)
+        elapsed=time.perf_counter()-t
+        ids=sequence[0,width:].tolist()
+        actual=[];ended=False
+        for tok in ids:
+            if tok==pad_id:break
+            actual.append(tok)
+            if tok in stop_ids:
+                ended=True;break
+        raw=tokenizer.decode(actual,skip_special_tokens=False)
+        parts=re.split(r'<\|channel>final|<channel\|>',raw)
+        final=re.sub(r'<[^>\n]+>','',parts[-1]).strip()
+        lines=[x.strip() for x in final.splitlines() if x.strip()]
+        last=lines[-1] if lines else ''
+        match=re.fullmatch(r'\*{0,2}\s*ANSWER\s*:\s*([ABCD])\s*[.!]?\s*\*{0,2}',last,flags=re.I)
+        pred=match.group(1).upper() if match else None
+        hit_limit=not ended and len(actual)>=TOKEN_BUDGET
+        valid=pred in LETTER_TO_NUMBER and ended and len(actual)>0
+        status=('completed' if valid else 'token_limit' if hit_limit else
+                'padding_only' if not actual else 'unparseable_or_unterminated')
+        return {'dataset_index':q['dataset_index'],'source_file':q['source_file'],
+                'source_position':q['source_position'],'input_fingerprint':fingerprint(q),
+                'prompt_sha256':PROMPT_SHA,'max_new_tokens':TOKEN_BUDGET,
+                'original_status':fail_by_idx[q['dataset_index']]['status'],
+                'native_thinking':True,'generated_tokens':len(actual),'seconds':round(elapsed,3),
+                'prediction':pred,'valid':bool(valid),'status':status,
+                'turn_end':turn_id in actual,'limit_hit':bool(hit_limit),
+                'last_line':last,'raw_output':raw,
+                'submission_record':{'q_no':q['q_no'],'subject':q['subject'],
+                    'category':q['category'],'level':q['level'],
+                    'answer':LETTER_TO_NUMBER[pred] if valid else None}}
+
+    print('\n▶ RETRY32-D — TARGETED SINGLE-QUESTION INFERENCE (4 HOURS)',flush=True)
+    start=time.perf_counter()
+    for idx in indices:
+        if retry_path(idx).exists():continue
+        if time.perf_counter()-start>=MAX_WALL_SECONDS:
+            print('Configured 4-hour inference limit reached',flush=True);break
+        try:
+            result=score_single(questions[idx])
+            validate_retry(result)
+            atomic_json(retry_path(idx),result)
+        except Exception:
+            checkpoint()
+            raise
+        state=checkpoint() # archive after every retry, not every 5
+        print(f'Index {idx} | status={result["status"]} | answer={result["submission_record"]["answer"]} '
+              f'| tokens={result["generated_tokens"]} | seconds={result["seconds"]:.1f} '
+              f'| retry valid={state["valid"]}/32',flush=True)
+
+state=checkpoint()
+print('\n=== E4B TARGETED RECOVERY STATUS ===')
+print('Previously valid: 2206')
+print('Retried:',state['attempted'],'/32')
+print('New valid:',state['valid'])
+print('Still failed:',state['still_invalid'])
+print('Not yet retried:',state['not_yet_retried'])
+print('Recovery ZIP:',ARCHIVE,flush=True)
+
+# Export only if every one of the 32 recoveries is valid.
+if state['attempted']==32 and state['valid']==32:
+    recovered=load_retries()
+    assert set(recovered)==set(indices)
+    combined=[]
+    for i in range(2238):
+        q=questions[i]
+        row=known_valid[i] if i in known_valid else recovered[i]['submission_record']
+        sub={field:row[field] for field in FIELDS}
+        assert list(sub)==FIELDS and sub['answer'] in ('1','2','3','4')
+        assert all(sub[k]==q[k] for k in FIELDS[:4])
+        combined.append(sub)
+    assert len(combined)==2238
+    # File ordering by canonical index is an internal reproducibility detail.
+    payload=(''.join(json.dumps(r,ensure_ascii=False,separators=(',',':'))+'\n' for r in combined)).encode('utf-8')
+    jpath=RUN/'predictions.jsonl'
+    jpath.write_bytes(payload)
+    with zipfile.ZipFile(FINAL_ZIP,'w',zipfile.ZIP_DEFLATED) as z:
+        z.writestr('predictions.jsonl',payload)
+    with zipfile.ZipFile(FINAL_ZIP) as z:
+        assert z.testzip() is None and z.namelist()==['predictions.jsonl']
+        lines=[json.loads(s) for s in z.read('predictions.jsonl').decode('utf-8').splitlines()]
+        assert lines==combined
+    atomic_json(RUN/'final_submission_audit.json',{
+        'total_rows':2238,'original_valid':2206,'recovered_valid':32,
+        'model':MODEL_REPO,'revision':REVISION,'prompt_sha256':PROMPT_SHA,
+        'submission_zip':str(FINAL_ZIP),'predictions_sha256':sha_file(jpath),
+        'validated_at':now(),'status':'READY_FOR_SUBMISSION'})
+    checkpoint()
+    print('▶ FINAL 2238/2238 SUBMISSION CREATED AND VERIFIED:',FINAL_ZIP,flush=True)
+else:
+    print('Submission ZIP NOT created. Resolve remaining invalid retries separately.',flush=True)
+print('▶ MIDDLE E4B RETRY32 COMPLETE',flush=True)
+

recovery/README.md

@@ -0,0 +1,20 @@
+# Historical inference and recovery evidence
+
+- `MIDDLE_E4B_RETRY32_COLDSTART.py` and `.ipynb`: original Kaggle RETRY32 recovery implementation, preserved as supplied.
+- `evidence/E4B_COMBINED_2206_WITH_FAILED32.zip`: the original 2,206-valid + 32-failed combined package. It contains records, failed raw outputs, merge audit, 41-file dataset SHA manifest and historical merge source.
+- `evidence/build_e4b_complete_firstpass.py`: the original merge script (preserved verbatim); it depends on three source shard ZIPs not included here.
+- `evidence/OFFLINE_CONSISTENCY_AUDIT.json`: independently computed offline audit of the supplied artifacts.
+
+**Verified:** All 2,206 first-pass valid rows are identical to the same-index rows in final `predictions.jsonl`. All 32 failed rows occupy the remaining unique positions and their question metadata matches the final submission. The first-pass merge audit and RETRY32 code specify the same V2 prompt SHA-256. This does **not** prove fresh GPU inference will produce exactly the same 2,238 outputs.
+
+## Optional source shard archives (not included)
+
+The first-pass historical merge audit gives these names and hashes; add only if available, with filenames preserved:
+
+| Source | Historical filename | SHA-256 |
+|---|---|---|
+| RIGHT | `LEFT_E4B_TEST_V2_RECOVERY_LATEST (6).zip` | `7a47cce96320d72ca3af31d2ac396b5621a3c113f3e68db2a9b0f786b31e3fb0` |
+| MIDDLE_TAIL500 | `MIDDLE_E4B_TAIL500_LATEST (2).zip` | `1ab6c129f48bea527cf9f1ea1be1b2e9b1e0bb1d954c1a02f01c89a788b16d6a` |
+| MIDDLE_GAP386 | `MIDDLE_E4B_GAP386_LATEST (2).zip` | `5e3db357b1a732b34c4b1f8ff3900f6c7220d13eac1eeb548fdb9aeea877ceee` |
+
+`recovery` files document historical inference; they are **not training artifacts** and do not provide precomputed answers to the organizer's standalone `inference.py`.