"""CPU-only, bounded-memory comparison of official and TensorAuto base tensors.""" import argparse,json,pathlib import torch from safetensors import safe_open def main(): p=argparse.ArgumentParser();p.add_argument('--official',required=True);p.add_argument('--external',required=True);p.add_argument('--output',required=True);a=p.parse_args() torch.set_num_threads(2);rows=[] excluded='paligemma_with_expert.gemma_expert.lm_head.weight' with safe_open(a.official,framework='pt',device='cpu') as src,safe_open(a.external,framework='pt',device='cpu') as dst: official_keys=set(src.keys());external_keys=set(dst.keys());metadata=src.metadata() or {} pairs=[(k,k) for k in sorted(official_keys) if k!=excluded] # Check extra serialized tied aliases too, rather than silently ignoring them. aliases={**metadata,'model.paligemma_with_expert.paligemma.model.language_model.embed_tokens.weight':'paligemma_with_expert.paligemma.lm_head.weight'} unmapped=[] for k in sorted(external_keys-official_keys): if k in aliases and aliases[k] in official_keys:pairs.append((aliases[k],k)) else:unmapped.append(k) for key,target in pairs: if target not in external_keys:rows.append(dict(key=key,target=target,status='missing'));continue xs=src.get_slice(key);ys=dst.get_slice(target);shape=xs.get_shape() if shape!=ys.get_shape():rows.append(dict(key=key,target=target,status='shape_mismatch'));continue exact=True;bf16=True;maximum=0.;dtypes=None chunk=max(1,4194304//max(1,torch.tensor(shape[1:]).prod().item())) for lo in range(0,shape[0],int(chunk)): x=xs[lo:lo+int(chunk)];y=ys[lo:lo+int(chunk)];dtypes=[str(x.dtype),str(y.dtype)] equal=torch.equal(x,y);exact &= equal bf16 &= torch.equal(x.bfloat16(),y.bfloat16()) if not equal:maximum=max(maximum,float((x.float()-y.float()).abs().max())) del x,y runtime_equal=bf16 if key.startswith('paligemma_with_expert.') else exact rows.append(dict(key=key,target=target,shape=shape,dtypes=dtypes,exact=exact,bf16_equal=bf16,current_loader_equal=runtime_equal,max_abs_difference=maximum)) result=dict(paired_experiment_uses_same_official_file=True,tensors=len(rows),all_exact=all(x.get('exact',False) for x in rows) and not unmapped,all_bf16_equal=all(x.get('bf16_equal',False) for x in rows) and not unmapped,all_current_loader_equal=all(x.get('current_loader_equal',False) for x in rows) and not unmapped,unmapped_external_keys=unmapped,rows=rows,unused_expert_vocabulary_head_excluded=True,external_revision='bff6ed8d0df75589e83f67a5cc282a1d6172efbc',external_sha256='feab100c33a77ec3cb310fa20591315903ba43b4ee10f318b55d9db1869bb7db') pathlib.Path(a.output).write_text(json.dumps(result,indent=2));print(json.dumps({k:v for k,v in result.items() if k!='rows'}),flush=True) if __name__=='__main__':main()