| |
| """Split a PEFT adapter_model.safetensors into nonce-prefixed .pt parts. |
| |
| WHY: HF hub (xet backend, hub 1.29) silently drops *.safetensors uploads to |
| this repo — upload_file/upload_folder return success and create a commit, |
| but the file 404s (safetensors scan path swallows it). Plain .pt/.json/.py |
| uploads are reliable. The smoke run (2026-09-22, session 4) hit the same |
| wall and invented this exact scheme; this script reproduces it. |
| |
| Usage (on the machine holding the adapter): |
| python3 t4_split_adapter.py <hf_lora_dir> # reads adapter_model.safetensors |
| Then upload <hf_lora_dir>/parts/* (parts + manifest.json + reassemble.py). |
| """ |
| import hashlib |
| import json |
| import os |
| import secrets |
| import sys |
|
|
| d = sys.argv[1] |
| src = os.path.join(d, "adapter_model.safetensors") |
| out = os.path.join(d, "parts") |
| os.makedirs(out, exist_ok=True) |
| data = open(src, "rb").read() |
| h = hashlib.sha256(data).hexdigest() |
| chunk = (len(data) + 3) // 4 |
| parts = [] |
| for i in range(4): |
| piece = data[i * chunk : (i + 1) * chunk] |
| name = f"adapter_p{i:02d}_{secrets.token_hex(4)}.pt" |
| open(os.path.join(out, name), "wb").write(secrets.token_bytes(16) + piece) |
| parts.append({"name": name, "size": len(piece) + 16}) |
| json.dump( |
| {"out_name": "adapter_model.safetensors", "sha256": h, "parts": parts, |
| "nonce_bytes": 16, "total_size": len(data)}, |
| open(os.path.join(out, "manifest.json"), "w"), indent=1) |
| print("SPLIT_OK", h, [p["name"] for p in parts]) |
|
|
