Download benchmark/revision.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 5.95 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/benchmark/revision.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/benchmark/revision.py
-
curl -L -o revision.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/benchmark/revision.py
5.95 kB
| """Explicit, hash-checked reuse after a data-only benchmark admission revision. | |
| Never overwrite the previous cohort or relabel changed inputs as old inference. | |
| Large immutable files are hard-linked on the same filesystem, not duplicated. | |
| """ | |
| import argparse | |
| import json | |
| import os | |
| import shutil | |
| from pathlib import Path | |
| from common import sha256, write_json | |
| def same_asset(row, asset): | |
| relation=row['overlay']; old=relation['external_benchmark'] | |
| return all(( | |
| relation['asset_id']==asset['asset_id'], row['voice']==asset['voice'], | |
| old['source_video_sha256']==asset['video_sha256'], | |
| old['feature_sha256']==asset['visual_features_sha256'], | |
| old['question_audio_sha256']==asset['question_audio_sha256'], | |
| old['review_receipt_sha256']==asset['visual_receipt_sha256'], | |
| old['question_time']==asset['question_time'], | |
| relation['question']==asset['question'], relation['answer']==asset['answer'], | |
| old['original_question']==asset['original_question'], | |
| old['original_answer']==asset['original_answer'], | |
| )) | |
| def link_checked(source, destination, digest): | |
| if sha256(source)!=digest:raise ValueError('Immutable input changed: '+str(source)) | |
| destination.parent.mkdir(parents=True,exist_ok=True) | |
| if destination.exists(): | |
| if sha256(destination)!=digest:raise ValueError('Conflicting revision file: '+str(destination)) | |
| else: | |
| os.link(source,destination) | |
| def seed_cohort(old, destination, identity, assets): | |
| manifest=json.loads((old/'manifest.json').read_text()) | |
| for key in ('run','run_config_sha256','count','seed','schema','max_qa_reuse'): | |
| if manifest[key]!=identity[key]:raise ValueError('Revision changes nonvisual selection condition: '+key) | |
| candidates={(r['asset_id'],r['voice']):r for r in assets} | |
| previous=json.loads((old/'selection_progress.json').read_text()) | |
| selected={r['sample_id'] for r in manifest['rows']} | |
| kept=[];removed=[] | |
| for row in previous['rows']: | |
| if row['sample_id'] not in selected:continue | |
| asset=candidates.get((row['overlay']['asset_id'],row['voice'])) | |
| if asset is None or not same_asset(row,asset): | |
| removed.append(row['sample_id']);continue | |
| for key in ('source_file','reference_file','microphone_file'): | |
| filename=row[key] | |
| if Path(filename).name!=filename:raise ValueError('Unsafe sample filename') | |
| link_checked(old/filename,destination/filename,row[key.replace('_file','_sha256')]) | |
| write_json(destination/(row['sample_id']+'.json'),row) | |
| kept.append(row) | |
| write_json(destination/'revision_provenance.json',dict( | |
| old_manifest_path=str((old/'manifest.json').resolve()),old_manifest_sha256=sha256(old/'manifest.json'), | |
| retained_input_identical=len(kept),removed_samples=removed, | |
| reason='Direct visual QA review only; no model-dependent selection')) | |
| return dict(identity=identity,rows=kept,rejections=[], | |
| attempted=[(r['dataset_index'],r['visual_kind']) for r in kept]) | |
| def reuse_generated(old_cohort,new_cohort,old_output,new_output): | |
| before=json.loads((old_cohort/'manifest.json').read_text()) | |
| after=json.loads((new_cohort/'manifest.json').read_text()) | |
| original_identity=json.loads((old_output/'identity.json').read_text()) | |
| if original_identity['cohort_sha256']!=sha256(old_cohort/'manifest.json'): | |
| raise ValueError('Original output belongs to a different cohort') | |
| if before['run']!=after['run'] or before['run_config_sha256']!=after['run_config_sha256']: | |
| raise ValueError('Checkpoint recipe changed') | |
| identity={**original_identity,'cohort_sha256':sha256(new_cohort/'manifest.json')} | |
| new_output.mkdir(parents=True,exist_ok=True) | |
| ip=new_output/'identity.json' | |
| if ip.exists() and json.loads(ip.read_text())!=identity:raise ValueError('Output identity collision') | |
| oldrows={r['sample_id']:r for r in before['rows']}; receipts=[] | |
| for row in after['rows']: | |
| sid=row['sample_id'];prior=oldrows.get(sid) | |
| if prior is None or any(row[k]!=prior[k] for k in ('source_sha256','reference_sha256','microphone_sha256')): | |
| continue | |
| meta=old_output/(sid+'.json');motion=old_output/(sid+'.npz') | |
| if not meta.exists() or not motion.exists():continue | |
| result=json.loads(meta.read_text()) | |
| if any(result.get(k)!=v for k,v in original_identity.items()):raise ValueError('Mixed original result') | |
| for key in ('source_file','reference_file','microphone_file'): | |
| if sha256(new_cohort/row[key])!=row[key.replace('_file','_sha256')]: | |
| raise ValueError('New cohort input changed') | |
| evidence=dict(original_result=str(meta.resolve()),original_result_sha256=sha256(meta), | |
| original_cohort_sha256=original_identity['cohort_sha256'],source_sha256=row['source_sha256'], | |
| reference_sha256=row['reference_sha256'],microphone_sha256=row['microphone_sha256']) | |
| link_checked(motion,new_output/motion.name,sha256(motion)) | |
| # Keep native audio/preview paths valid; original artifacts are immutable evidence. | |
| folder=old_output/sid;newfolder=new_output/sid | |
| if folder.is_dir() and not newfolder.exists(): | |
| shutil.copytree(folder,newfolder,copy_function=os.link) | |
| write_json(new_output/meta.name,{**result,**identity,'reuse_provenance':evidence}) | |
| receipts.append(dict(sample_id=sid,**evidence)) | |
| write_json(ip,identity) | |
| write_json(new_output/'reuse_receipts.json',dict(count=len(receipts),rows=receipts)) | |
| print('REUSED_GENERATED',len(receipts),flush=True) | |
| if __name__=='__main__': | |
| p=argparse.ArgumentParser(description=__doc__) | |
| for key in ('old-cohort','new-cohort','old-output','new-output'): | |
| p.add_argument('--'+key,type=Path,required=True) | |
| a=p.parse_args();reuse_generated(a.old_cohort,a.new_cohort,a.old_output,a.new_output) | |