Download scale/script/audit_reference.py from Tinghong-Ye22/code-Movens-Zero_train: direct link, hf CLI and curl.
- Browser
- Download file 7.46 kB
-
https://huggingface.co/Tinghong-Ye22/code-Movens-Zero_train/resolve/main/scale/script/audit_reference.py
- Command line
-
hf download hf://Tinghong-Ye22/code-Movens-Zero_train/scale/script/audit_reference.py
-
curl -L -o audit_reference.py https://huggingface.co/Tinghong-Ye22/code-Movens-Zero_train/resolve/main/scale/script/audit_reference.py
7.46 kB
| """Read-only reference comparison; reports are written only under local scale/audits.""" | |
| import csv | |
| import difflib | |
| import hashlib | |
| import json | |
| from pathlib import Path | |
| import sys | |
| ROOT = Path(__file__).resolve().parents[2] | |
| WS = ROOT.parents[2] | |
| ORIGINAL = WS/'buwendong/user/code/Movens-Zero_train' | |
| UPSTREAM = WS/'buwendong/user/code/OmniHM-Zero_train_40d' | |
| GROUP = ROOT/'scale/groups/sml_1m_224_a1e-4_o1e-5_nojerk' | |
| OUT = ROOT/'scale/audits/reference_20261002' | |
| sys.path.insert(0, str(ROOT/'scale/script')) | |
| from preview_scale import save | |
| def diff(a, b, prefix=''): | |
| result = {} | |
| if isinstance(a, dict) and isinstance(b, dict): | |
| for key in sorted(a.keys() | b.keys()): | |
| result.update(diff(a.get(key), b.get(key), f'{prefix}.{key}' if prefix else key)) | |
| elif a != b: | |
| result[prefix] = {'reference': a, 'current': b} | |
| return result | |
| def sha(p): | |
| return hashlib.sha256(p.read_bytes()).hexdigest() | |
| def main(): | |
| OUT.mkdir(parents=True, exist_ok=True) | |
| paths = set(p.relative_to(ROOT) for folder in ('src','scripts') for p in (ROOT/folder).rglob('*.py')) | |
| paths.update(Path(p) for p in ('train_vlm.py','start_visualization.py','prepare_training_set.py')) | |
| files = []; patches = []; upstream_patches = [] | |
| for relative in sorted(paths): | |
| p, q = ROOT/relative, ORIGINAL/relative | |
| item = dict(path=str(relative), current_sha256=sha(p), reference_exists=q.is_file()) | |
| if q.is_file(): | |
| item.update(reference_sha256=sha(q), identical=p.read_bytes()==q.read_bytes()) | |
| if not item['identical']: | |
| patches.extend(difflib.unified_diff(q.read_text().splitlines(True),p.read_text().splitlines(True), | |
| fromfile=f'original/{relative}',tofile=f'current/{relative}')) | |
| upstream = UPSTREAM/relative | |
| if upstream.is_file(): | |
| item.update(upstream_sha256=sha(upstream), upstream_identical=upstream.read_bytes()==p.read_bytes()) | |
| if not item['upstream_identical']: | |
| upstream_patches.extend(difflib.unified_diff(upstream.read_text().splitlines(True),p.read_text().splitlines(True), | |
| fromfile=f'upstream_current/{relative}',tofile=f'current/{relative}')) | |
| files.append(item) | |
| save(OUT/'code_hashes.json', files) | |
| (OUT/'core_changes.diff').write_text(''.join(patches)) | |
| (OUT/'upstream_live_changes.diff').write_text(''.join(upstream_patches)) | |
| launchers=[] | |
| for p in sorted(ORIGINAL.glob('train*')): | |
| if p.suffix not in ('.py','.sh') or not p.is_file():continue | |
| target=ROOT/p.name | |
| launchers.append(dict(name=p.name,exists=target.exists(),identical=target.exists() and sha(p)==sha(target))) | |
| save(OUT/'original_launcher_identity.json',launchers) | |
| from train_scale import normal_recipe | |
| normal, prepared = normal_recipe() | |
| reference = normal.model_dump(mode='json') | |
| save(OUT/'reconstructed_normal_startup.json',reference) | |
| configs={} | |
| for size in 'SML': | |
| run=Path(json.loads((GROUP/f'runs/latest_{size}.json').read_text())['run_dir']) | |
| config=json.loads((run/'run_config.json').read_text()) | |
| configs[size]=config | |
| save(OUT/f'{size}_config_differences.json',diff(reference,config)) | |
| save(OUT/f'{size}_run_config.json',config) | |
| initial_path=sorted((GROUP/'launches').glob(f'{size}_260927*/launch.json'))[0] | |
| initial=json.loads(initial_path.read_text()) | |
| save(OUT/f'{size}_initial_launch.json',initial) | |
| save(OUT/f'{size}_initial_config_differences.json',diff(reference,initial)) | |
| integrity=json.loads((ROOT/'training_set/integrity.json').read_text()) | |
| manifests=[] | |
| for key,relative in integrity['manifests'].items(): | |
| p=ROOT/'training_set'/relative; q=ORIGINAL/'training_set'/relative | |
| if p.suffix=='.csv': | |
| with p.open() as f: rows=list(csv.DictReader(f)) | |
| with q.open() as f: other=list(csv.DictReader(f)) | |
| else: | |
| rows=[json.loads(line) for line in p.read_text().splitlines() if line.strip()] | |
| other=[json.loads(line) for line in q.read_text().splitlines() if line.strip()] | |
| manifests.append(dict(name=key,rows=len(rows),reference_rows=len(other), | |
| byte_identical=sha(p)==sha(q), parsed_identical=rows==other, | |
| equal_after_project_path_relocation=(json.dumps(rows,sort_keys=True).replace(str(ROOT),'<PROJECT>') == | |
| json.dumps(other,sort_keys=True).replace(str(ORIGINAL),'<PROJECT>')), | |
| changed_fields=sorted({field for a,b in zip(rows,other) for field in a.keys()|b.keys() if a.get(field)!=b.get(field)}), | |
| current_sha256=sha(p), reference_sha256=sha(q))) | |
| save(OUT/'data_manifests.json',manifests) | |
| paths=[Path('training_set/config/training_recipe.json'),Path('training_set/config/representation.json')] | |
| save(OUT/'data_recipe_differences.json',{str(p):diff(json.loads((ORIGINAL/p).read_text()),json.loads((ROOT/p).read_text())) for p in paths}) | |
| seeds={label:json.loads(Path(cfg['train']['absolute_height_startup_seed_path']).read_text()) | |
| for label,cfg in {'reference':reference,**configs}.items()} | |
| save(OUT/'startup_seed_differences.json',{size:diff(seeds['reference'],seeds[size]) for size in 'SML'}) | |
| stats={name:json.loads(Path(cfg['data']['transform']['field']['stat_path']).read_text()) | |
| for name,cfg in {'reference':reference,**configs}.items()} | |
| save(OUT/'normalization_differences.json',{size:diff(stats['reference'],stats[size]) for size in 'SML'}) | |
| source_diff={} | |
| for size,cfg in configs.items(): | |
| source_diff[size]={} | |
| for task in ('text','audio','audio_command','video_clip'): | |
| a=json.loads(reference['data'][task]['sources_json']); b=json.loads(cfg['data'][task]['sources_json']) | |
| for x in a+b: | |
| x.pop('records_cache_dir',None) | |
| source_diff[size][task]=diff(a,b) | |
| save(OUT/'data_source_semantics_differences.json',source_diff) | |
| chain=[]; checkpoint=Path(json.loads((ROOT/'config/runtime.json').read_text())['resume_checkpoint']) | |
| seen=set() | |
| while checkpoint.parent.parent not in seen: | |
| run=checkpoint.parent.parent; seen.add(run) | |
| path=run/'run_config.json' | |
| if not path.exists(): | |
| chain.append(dict(run=str(run),missing_config=True)); break | |
| cfg=json.loads(path.read_text()) | |
| entry=dict(run=str(run),checkpoint=checkpoint.name,config_sha256=sha(path), | |
| training_protocol=cfg.get('training_protocol'),stage=cfg.get('absolute_height_stage'), | |
| base_action_lr=cfg['train']['learning_rate'],base_omni_lr=cfg['model']['lang_backbone_lr'], | |
| global_batch=cfg['train']['train_batch_size']*cfg['world_size_expected'], | |
| jerk_weights={k:v for k,v in cfg['train'].items() if 'jerk' in k and 'weight' in k}) | |
| chain.append(entry) | |
| provenance=cfg.get('resume_provenance') or cfg.get('warmstart_provenance') or {} | |
| previous=provenance.get('source_checkpoint') | |
| if not previous: break | |
| checkpoint=Path(previous) | |
| save(OUT/'reference_checkpoint_chain.json',chain) | |
| print(json.dumps(dict(code_files=len(files),identical=sum(x.get('identical',False) for x in files), | |
| changed=[x['path'] for x in files if x.get('identical') is False], | |
| manifests=manifests),ensure_ascii=False,indent=2)) | |
| if __name__=='__main__': | |
| main() | |