code-HFD / benchmark /revision.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
fb0074a verified
Raw History Blame Contribute Delete
5.95 kB
"""Explicit, hash-checked reuse after a data-only benchmark admission revision.
Never overwrite the previous cohort or relabel changed inputs as old inference.
Large immutable files are hard-linked on the same filesystem, not duplicated.
"""
import argparse
import json
import os
import shutil
from pathlib import Path
from common import sha256, write_json
def same_asset(row, asset):
relation=row['overlay']; old=relation['external_benchmark']
return all((
relation['asset_id']==asset['asset_id'], row['voice']==asset['voice'],
old['source_video_sha256']==asset['video_sha256'],
old['feature_sha256']==asset['visual_features_sha256'],
old['question_audio_sha256']==asset['question_audio_sha256'],
old['review_receipt_sha256']==asset['visual_receipt_sha256'],
old['question_time']==asset['question_time'],
relation['question']==asset['question'], relation['answer']==asset['answer'],
old['original_question']==asset['original_question'],
old['original_answer']==asset['original_answer'],
))
def link_checked(source, destination, digest):
if sha256(source)!=digest:raise ValueError('Immutable input changed: '+str(source))
destination.parent.mkdir(parents=True,exist_ok=True)
if destination.exists():
if sha256(destination)!=digest:raise ValueError('Conflicting revision file: '+str(destination))
else:
os.link(source,destination)
def seed_cohort(old, destination, identity, assets):
manifest=json.loads((old/'manifest.json').read_text())
for key in ('run','run_config_sha256','count','seed','schema','max_qa_reuse'):
if manifest[key]!=identity[key]:raise ValueError('Revision changes nonvisual selection condition: '+key)
candidates={(r['asset_id'],r['voice']):r for r in assets}
previous=json.loads((old/'selection_progress.json').read_text())
selected={r['sample_id'] for r in manifest['rows']}
kept=[];removed=[]
for row in previous['rows']:
if row['sample_id'] not in selected:continue
asset=candidates.get((row['overlay']['asset_id'],row['voice']))
if asset is None or not same_asset(row,asset):
removed.append(row['sample_id']);continue
for key in ('source_file','reference_file','microphone_file'):
filename=row[key]
if Path(filename).name!=filename:raise ValueError('Unsafe sample filename')
link_checked(old/filename,destination/filename,row[key.replace('_file','_sha256')])
write_json(destination/(row['sample_id']+'.json'),row)
kept.append(row)
write_json(destination/'revision_provenance.json',dict(
old_manifest_path=str((old/'manifest.json').resolve()),old_manifest_sha256=sha256(old/'manifest.json'),
retained_input_identical=len(kept),removed_samples=removed,
reason='Direct visual QA review only; no model-dependent selection'))
return dict(identity=identity,rows=kept,rejections=[],
attempted=[(r['dataset_index'],r['visual_kind']) for r in kept])
def reuse_generated(old_cohort,new_cohort,old_output,new_output):
before=json.loads((old_cohort/'manifest.json').read_text())
after=json.loads((new_cohort/'manifest.json').read_text())
original_identity=json.loads((old_output/'identity.json').read_text())
if original_identity['cohort_sha256']!=sha256(old_cohort/'manifest.json'):
raise ValueError('Original output belongs to a different cohort')
if before['run']!=after['run'] or before['run_config_sha256']!=after['run_config_sha256']:
raise ValueError('Checkpoint recipe changed')
identity={**original_identity,'cohort_sha256':sha256(new_cohort/'manifest.json')}
new_output.mkdir(parents=True,exist_ok=True)
ip=new_output/'identity.json'
if ip.exists() and json.loads(ip.read_text())!=identity:raise ValueError('Output identity collision')
oldrows={r['sample_id']:r for r in before['rows']}; receipts=[]
for row in after['rows']:
sid=row['sample_id'];prior=oldrows.get(sid)
if prior is None or any(row[k]!=prior[k] for k in ('source_sha256','reference_sha256','microphone_sha256')):
continue
meta=old_output/(sid+'.json');motion=old_output/(sid+'.npz')
if not meta.exists() or not motion.exists():continue
result=json.loads(meta.read_text())
if any(result.get(k)!=v for k,v in original_identity.items()):raise ValueError('Mixed original result')
for key in ('source_file','reference_file','microphone_file'):
if sha256(new_cohort/row[key])!=row[key.replace('_file','_sha256')]:
raise ValueError('New cohort input changed')
evidence=dict(original_result=str(meta.resolve()),original_result_sha256=sha256(meta),
original_cohort_sha256=original_identity['cohort_sha256'],source_sha256=row['source_sha256'],
reference_sha256=row['reference_sha256'],microphone_sha256=row['microphone_sha256'])
link_checked(motion,new_output/motion.name,sha256(motion))
# Keep native audio/preview paths valid; original artifacts are immutable evidence.
folder=old_output/sid;newfolder=new_output/sid
if folder.is_dir() and not newfolder.exists():
shutil.copytree(folder,newfolder,copy_function=os.link)
write_json(new_output/meta.name,{**result,**identity,'reuse_provenance':evidence})
receipts.append(dict(sample_id=sid,**evidence))
write_json(ip,identity)
write_json(new_output/'reuse_receipts.json',dict(count=len(receipts),rows=receipts))
print('REUSED_GENERATED',len(receipts),flush=True)
if __name__=='__main__':
p=argparse.ArgumentParser(description=__doc__)
for key in ('old-cohort','new-cohort','old-output','new-output'):
p.add_argument('--'+key,type=Path,required=True)
a=p.parse_args();reuse_generated(a.old_cohort,a.new_cohort,a.old_output,a.new_output)