File size: 1,021 Bytes
1f78dbc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 | import polars as pl, numpy as np, json
V='/workspace/huawei/v2work'; L='/workspace/huawei/parallel/lag'; out={}
for d,a,b in (('feat_v3','val25','val25repro'),('feat','val25','val25repro'),('feat_x','val25','val25'),('feat_x2','val25','val25')):
A=pl.read_parquet(f'{V}/{d}/{a}.parquet'); B=pl.read_parquet(f'{L}/{d}/{b}.parquet')
B=A.select('rowid').join(B,on='rowid',how='left')
cols=[c for c in A.columns if c in B.columns and c!='rowid' and A[c].dtype.is_numeric()]
bad={}
for c in cols:
x=A[c].cast(pl.Float64).to_numpy(); y=B[c].cast(pl.Float64).to_numpy()
dif=np.nanmax(np.abs(x-y)) if np.isfinite(x-y).any() else 0.0
nanm=int((np.isnan(x)!=np.isnan(y)).sum())
if dif>1e-3*max(1,np.nanmax(np.abs(x))) or nanm: bad[c]=(float(dif),nanm)
out[d]=dict(ncols=len(cols),only_disk=[c for c in A.columns if c not in B.columns],n_diff=len(bad),diff=dict(list(bad.items())[:12]))
print(json.dumps(out,indent=1)); json.dump(out,open(f'{L}/repro_cmp.json','w'),indent=1)
|