Download validate.py from gatilin/bitdance_binary_tokenizer_d32c128: direct link, hf CLI and curl.
- Browser
- Download file 2.49 kB
-
https://huggingface.co/gatilin/bitdance_binary_tokenizer_d32c128/resolve/main/validate.py
- Command line
-
hf download hf://gatilin/bitdance_binary_tokenizer_d32c128/validate.py
-
curl -L -o validate.py https://huggingface.co/gatilin/bitdance_binary_tokenizer_d32c128/resolve/main/validate.py
2.49 kB
| from pathlib import Path | |
| import json,time,torch,hashlib | |
| from component import load_model,pack_bits,unpack_bits | |
| root=Path(__file__).parent;torch.set_num_threads(4);checks=[];t=time.time() | |
| def check(name,ok,**details): | |
| assert ok,(name,details) | |
| checks.append(dict(name=name,passed=True,**details));print(name,details,flush=True) | |
| m=load_model(root);check("strict_load",True,tensors=len(m.state_dict())) | |
| fixture=torch.linspace(-1,1,3*256*256).reshape(1,3,256,256) | |
| with torch.inference_mode(): | |
| latent=m.encode(fixture);check("encode_256",latent.shape==(1,128,8,8) and torch.isfinite(latent).all().item(),shape=list(latent.shape)) | |
| check("binary_alphabet",bool(torch.all((latent==1)|(latent==-1)))) | |
| packed=pack_bits(latent);restored=unpack_bits(packed);check("portable_bit_roundtrip",torch.equal(latent,restored),packed_shape=list(packed.shape)) | |
| torch.manual_seed(1019);decoded=m.decode(latent);torch.manual_seed(1019);from_bits=m.decode(restored);check("decode_bit_parity",torch.equal(decoded,from_bits),max_abs_error=(decoded-from_bits).abs().max().item()) | |
| torch.manual_seed(2026);alternative=m.decode(latent);check("native_stochastic_decoder",not torch.equal(decoded,alternative),max_abs_difference=(decoded-alternative).abs().max().item()) | |
| check("reconstruction_256",decoded.shape==fixture.shape and torch.isfinite(decoded).all().item(),shape=list(decoded.shape),mse=(decoded-fixture).square().mean().item()) | |
| x=torch.linspace(-1,1,3*256*320).reshape(1,3,256,320);z=m.encode(x);y=m.decode(z) | |
| check("rectangular_grid",z.shape==(1,128,8,10) and y.shape==x.shape and torch.isfinite(y).all().item(),latent_shape=list(z.shape),image_shape=list(y.shape)) | |
| try:pack_bits(torch.zeros(1,128,2,2));raise AssertionError("accepted invalid signs") | |
| except ValueError:check("reject_invalid_signs",True) | |
| source=json.loads((root/"model.safetensors.provenance.json").read_text());h=hashlib.file_digest((root/"model.safetensors").open("rb"),"sha256").hexdigest();check("source_LFS_sha256",h==source["sha256"]) | |
| report=dict(strict_load=True,source_tensor_bytes_equal=True,checks=checks,smoke=dict(passed=True,device="cpu",dtype="float32"),elapsed_seconds=time.time()-t,limitations=["Portable bit helper is explicitly separate from any upstream LLM token mapping.","No BitDance text-to-image backbone or UniWeTok semantic checkpoint tested."]) | |
| (root/"validation.json").write_text(json.dumps(report,indent=2)+"\n");print("BITDANCE VALIDATED",report["elapsed_seconds"],flush=True) | |