File size: 1,049 Bytes
67ebcef | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 | """Writes a llama.cpp control-vector GGUF: alpha * unit(direction[k]) * avg_norm[k] added to the output of each layer
in --layers (0-based; layer 0 cannot be steered by llama.cpp). Negative alpha = steer AWAY from the direction.
usage: make_cvec.py results/cvec/hes.pt out.gguf --alpha -0.1 --layers 12 13 14"""
import argparse, torch, gguf
ap = argparse.ArgumentParser()
ap.add_argument("src"); ap.add_argument("out")
ap.add_argument("--alpha", type=float, required=True)
ap.add_argument("--layers", type=int, nargs="+", required=True)
a = ap.parse_args()
s = torch.load(a.src)
d, nrm = s["dir"], s["avg_norm"]
w = gguf.GGUFWriter(a.out, "controlvector")
w.add_string("controlvector.model_hint", "lfm3")
w.add_uint32("controlvector.layer_count", len(a.layers))
for k in a.layers:
assert k >= 1
v = a.alpha * nrm[k] * d[k] / d[k].norm()
w.add_tensor(f"direction.{k}", v.float().numpy())
w.write_header_to_file(); w.write_kv_data_to_file(); w.write_tensors_to_file(); w.close()
print("wrote", a.out, "alpha", a.alpha, "layers", a.layers)
|