Upload 9 files
Browse files- examples/reproduce_gpt2_bengali.py +79 -0
- examples/train_with_trainer.py +47 -0
- src/tensorpeft/__init__.py +16 -0
- src/tensorpeft/config.py +69 -0
- src/tensorpeft/cp.py +66 -0
- src/tensorpeft/layers.py +103 -0
- src/tensorpeft/model.py +307 -0
- src/tensorpeft/trainer.py +18 -0
- tests/test_tensorpeft.py +216 -0
examples/reproduce_gpt2_bengali.py
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Reproduce the GPT-2 / Bengali small-budget benchmark with the package API (plain PyTorch loop).
|
| 2 |
+
|
| 3 |
+
python reproduce_gpt2_bengali.py --model <gpt2 dir or 'gpt2'> --data <dir with train.bin, val.bin> \
|
| 4 |
+
--atoms-per-group 1 # 1 -> 9,259 params, 4 -> 37,036, 6 -> 55,554
|
| 5 |
+
|
| 6 |
+
train.bin / val.bin are uint16 GPT-2 token ids. Reported number: mean cross-entropy (nats/token)
|
| 7 |
+
on the first 64 windows of 128+1 tokens of val.bin.
|
| 8 |
+
"""
|
| 9 |
+
import argparse
|
| 10 |
+
import math
|
| 11 |
+
import time
|
| 12 |
+
|
| 13 |
+
import numpy as np
|
| 14 |
+
import torch
|
| 15 |
+
import torch.nn.functional as F
|
| 16 |
+
from transformers import AutoModelForCausalLM
|
| 17 |
+
|
| 18 |
+
from tensorpeft import TensorPEFTConfig, get_tensorpeft_model
|
| 19 |
+
|
| 20 |
+
p = argparse.ArgumentParser()
|
| 21 |
+
p.add_argument("--model", required=True)
|
| 22 |
+
p.add_argument("--data", required=True)
|
| 23 |
+
p.add_argument("--atoms-per-group", type=int, default=1)
|
| 24 |
+
p.add_argument("--steps", type=int, default=300)
|
| 25 |
+
p.add_argument("--lr", type=float, default=3e-3)
|
| 26 |
+
p.add_argument("--warmup", type=int, default=10)
|
| 27 |
+
p.add_argument("--batch", type=int, default=8)
|
| 28 |
+
p.add_argument("--seq", type=int, default=128)
|
| 29 |
+
p.add_argument("--init-batches", type=int, default=4)
|
| 30 |
+
p.add_argument("--seed", type=int, default=0)
|
| 31 |
+
p.add_argument("--save", default=None)
|
| 32 |
+
a = p.parse_args()
|
| 33 |
+
torch.manual_seed(a.seed)
|
| 34 |
+
|
| 35 |
+
train = np.memmap(f"{a.data}/train.bin", dtype=np.uint16, mode="r")
|
| 36 |
+
val = np.memmap(f"{a.data}/val.bin", dtype=np.uint16, mode="r")
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def sampler(seed):
|
| 40 |
+
rng = np.random.default_rng(seed)
|
| 41 |
+
while True:
|
| 42 |
+
ix = rng.integers(0, len(train) - a.seq - 1, a.batch)
|
| 43 |
+
yield torch.from_numpy(np.stack([train[i:i + a.seq + 1] for i in ix]).astype(np.int64))
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
V = torch.from_numpy(np.asarray(val[:64 * (a.seq + 1)]).astype(np.int64).reshape(64, a.seq + 1))
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def loss_on(model, x):
|
| 50 |
+
return F.cross_entropy(model(x[:, :-1]).logits.flatten(0, 1), x[:, 1:].flatten())
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
@torch.no_grad()
|
| 54 |
+
def evaluate(model):
|
| 55 |
+
return sum(float(loss_on(model, V[i:i + 16])) * len(V[i:i + 16]) for i in range(0, len(V), 16)) / len(V)
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
base = AutoModelForCausalLM.from_pretrained(a.model, dtype=torch.float32).eval()
|
| 59 |
+
model = get_tensorpeft_model(base, TensorPEFTConfig(atoms_per_group=a.atoms_per_group, init_batches=a.init_batches))
|
| 60 |
+
print("base val", round(evaluate(model), 4))
|
| 61 |
+
t0 = time.time()
|
| 62 |
+
model.init_atoms(sampler(a.seed + 12345)) # one gradient probe -> all atoms
|
| 63 |
+
model.print_trainable_parameters()
|
| 64 |
+
|
| 65 |
+
opt = torch.optim.AdamW([q for q in model.parameters() if q.requires_grad], lr=a.lr, weight_decay=0.0)
|
| 66 |
+
sched = torch.optim.lr_scheduler.LambdaLR(opt, lambda s: min(1.0, (s + 1) / a.warmup))
|
| 67 |
+
batches = sampler(a.seed)
|
| 68 |
+
for step in range(1, a.steps + 1):
|
| 69 |
+
loss = loss_on(model, next(batches))
|
| 70 |
+
opt.zero_grad(set_to_none=True)
|
| 71 |
+
loss.backward()
|
| 72 |
+
opt.step()
|
| 73 |
+
sched.step()
|
| 74 |
+
if step % 60 == 0 or step == a.steps:
|
| 75 |
+
print(f"step {step} train {float(loss):.4f} val {evaluate(model):.4f} {time.time() - t0:.0f}s", flush=True)
|
| 76 |
+
v = evaluate(model)
|
| 77 |
+
print(f"final val {v:.4f} (ppl {math.exp(v):.2f})")
|
| 78 |
+
if a.save:
|
| 79 |
+
model.save_pretrained(a.save)
|
examples/train_with_trainer.py
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Fine-tune any Hugging Face causal LM on a text file with TensorPEFT and the HF Trainer.
|
| 2 |
+
|
| 3 |
+
python train_with_trainer.py --model Qwen/Qwen2.5-0.5B --text corpus.txt --out tp_adapter \
|
| 4 |
+
--atoms-per-group 6 --steps 300
|
| 5 |
+
"""
|
| 6 |
+
import argparse
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer, DataCollatorForLanguageModeling, TrainingArguments
|
| 10 |
+
|
| 11 |
+
from tensorpeft import TensorPEFTConfig, TensorPEFTTrainer, get_tensorpeft_model
|
| 12 |
+
|
| 13 |
+
p = argparse.ArgumentParser()
|
| 14 |
+
p.add_argument("--model", required=True)
|
| 15 |
+
p.add_argument("--text", required=True, help="plain-text training file")
|
| 16 |
+
p.add_argument("--out", required=True)
|
| 17 |
+
p.add_argument("--atoms-per-group", type=int, default=4)
|
| 18 |
+
p.add_argument("--seq", type=int, default=256)
|
| 19 |
+
p.add_argument("--batch", type=int, default=8)
|
| 20 |
+
p.add_argument("--steps", type=int, default=300)
|
| 21 |
+
p.add_argument("--lr", type=float, default=3e-3)
|
| 22 |
+
p.add_argument("--bf16", action="store_true")
|
| 23 |
+
a = p.parse_args()
|
| 24 |
+
|
| 25 |
+
tok = AutoTokenizer.from_pretrained(a.model)
|
| 26 |
+
if tok.pad_token is None:
|
| 27 |
+
tok.pad_token = tok.eos_token
|
| 28 |
+
model = AutoModelForCausalLM.from_pretrained(a.model, dtype=torch.bfloat16 if a.bf16 else torch.float32)
|
| 29 |
+
|
| 30 |
+
ids = tok(open(a.text, encoding="utf-8").read(), return_tensors="pt").input_ids[0]
|
| 31 |
+
n = len(ids) // a.seq
|
| 32 |
+
blocks = ids[: n * a.seq].view(n, a.seq)
|
| 33 |
+
train = [{"input_ids": b} for b in blocks]
|
| 34 |
+
|
| 35 |
+
model = get_tensorpeft_model(model, TensorPEFTConfig(atoms_per_group=a.atoms_per_group))
|
| 36 |
+
collate = DataCollatorForLanguageModeling(tok, mlm=False)
|
| 37 |
+
loader = torch.utils.data.DataLoader(train, batch_size=a.batch, shuffle=True, collate_fn=collate)
|
| 38 |
+
model.init_atoms(loader, n_batches=4) # the one extra step compared with LoRA
|
| 39 |
+
model.print_trainable_parameters()
|
| 40 |
+
|
| 41 |
+
args = TrainingArguments(output_dir=a.out, per_device_train_batch_size=a.batch, max_steps=a.steps,
|
| 42 |
+
learning_rate=a.lr, warmup_steps=10, lr_scheduler_type="constant_with_warmup",
|
| 43 |
+
weight_decay=0.0, logging_steps=20, save_steps=100, bf16=a.bf16, report_to=[])
|
| 44 |
+
trainer = TensorPEFTTrainer(model=model, args=args, train_dataset=train, data_collator=collate)
|
| 45 |
+
trainer.train()
|
| 46 |
+
model.save_pretrained(a.out) # adapter only (tens of KB for small budgets)
|
| 47 |
+
# model.push_to_hub("your-name/your-adapter")
|
src/tensorpeft/__init__.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 Angshul Majumdar. Licensed under the Apache License, Version 2.0.
|
| 2 |
+
"""TensorPEFT: parameter-efficient fine-tuning with tensor gradient-pursuit adapters."""
|
| 3 |
+
__version__ = "0.1.0"
|
| 4 |
+
|
| 5 |
+
from .config import TensorPEFTConfig # noqa: E402
|
| 6 |
+
from .model import TensorPEFTModel, get_tensorpeft_model # noqa: E402
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def __getattr__(name): # lazy: importing the Trainer pulls in accelerate
|
| 10 |
+
if name == "TensorPEFTTrainer":
|
| 11 |
+
from .trainer import TensorPEFTTrainer
|
| 12 |
+
return TensorPEFTTrainer
|
| 13 |
+
raise AttributeError(name)
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
__all__ = ["TensorPEFTConfig", "TensorPEFTModel", "TensorPEFTTrainer", "get_tensorpeft_model", "__version__"]
|
src/tensorpeft/config.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 Angshul Majumdar. Licensed under the Apache License, Version 2.0.
|
| 2 |
+
"""TensorPEFTConfig."""
|
| 3 |
+
import json
|
| 4 |
+
import os
|
| 5 |
+
from dataclasses import asdict, dataclass, field
|
| 6 |
+
from typing import List, Optional, Union
|
| 7 |
+
|
| 8 |
+
CONFIG_NAME = "adapter_config.json"
|
| 9 |
+
WEIGHTS_NAME = "adapter_model.safetensors"
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
@dataclass
|
| 13 |
+
class TensorPEFTConfig:
|
| 14 |
+
"""Configuration of a TensorPEFT adapter.
|
| 15 |
+
|
| 16 |
+
Args:
|
| 17 |
+
atoms_per_group: rank-one CP atoms per tensor group (all chosen at initialisation).
|
| 18 |
+
Trainable parameters = sum over groups of atoms_per_group * (1 + sum of the group's mode sizes).
|
| 19 |
+
target_modules: ``"all-linear"`` (every Linear / Conv1D inside the transformer blocks) or a
|
| 20 |
+
list of module-name suffixes, e.g. ``["q_proj", "v_proj"]`` (``"c_attn.q"`` etc. for GPT-2).
|
| 21 |
+
exclude_modules: substrings of module names that are never adapted.
|
| 22 |
+
split_fused_qkv: split GPT-2-style fused ``c_attn`` = [q|k|v] into q, k, v.
|
| 23 |
+
hopm_iters: higher-order power iterations used to compute each atom.
|
| 24 |
+
alpha_scale: coefficients are parametrised as ``alpha_scale * alpha_raw`` so that a single
|
| 25 |
+
optimiser learning rate (about 3e-3 for Adam/AdamW) suits both coefficients and factors.
|
| 26 |
+
init_batches: default number of batches used by ``init_atoms``.
|
| 27 |
+
"""
|
| 28 |
+
atoms_per_group: int = 4
|
| 29 |
+
target_modules: Union[str, List[str]] = "all-linear"
|
| 30 |
+
exclude_modules: List[str] = field(default_factory=lambda: ["lm_head", "embed", "wte", "wpe"])
|
| 31 |
+
split_fused_qkv: bool = True
|
| 32 |
+
hopm_iters: int = 30
|
| 33 |
+
alpha_scale: float = 100.0 / 3.0
|
| 34 |
+
init_batches: int = 4
|
| 35 |
+
base_model_name_or_path: Optional[str] = None
|
| 36 |
+
peft_type: str = "TENSORPEFT"
|
| 37 |
+
task_type: str = "CAUSAL_LM"
|
| 38 |
+
tensorpeft_version: Optional[str] = None
|
| 39 |
+
|
| 40 |
+
def to_dict(self):
|
| 41 |
+
return asdict(self)
|
| 42 |
+
|
| 43 |
+
def save_pretrained(self, save_directory, extra=None):
|
| 44 |
+
os.makedirs(save_directory, exist_ok=True)
|
| 45 |
+
d = self.to_dict()
|
| 46 |
+
if extra:
|
| 47 |
+
d.update(extra)
|
| 48 |
+
with open(os.path.join(save_directory, CONFIG_NAME), "w") as f:
|
| 49 |
+
json.dump(d, f, indent=2)
|
| 50 |
+
|
| 51 |
+
@classmethod
|
| 52 |
+
def from_dict(cls, d):
|
| 53 |
+
names = {f for f in cls.__dataclass_fields__}
|
| 54 |
+
return cls(**{k: v for k, v in d.items() if k in names})
|
| 55 |
+
|
| 56 |
+
@classmethod
|
| 57 |
+
def from_pretrained(cls, model_id, **hub_kwargs):
|
| 58 |
+
path = _resolve(model_id, CONFIG_NAME, **hub_kwargs)
|
| 59 |
+
with open(path) as f:
|
| 60 |
+
d = json.load(f)
|
| 61 |
+
return cls.from_dict(d), d
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def _resolve(model_id, filename, **hub_kwargs):
|
| 65 |
+
local = os.path.join(model_id, filename)
|
| 66 |
+
if os.path.isfile(local):
|
| 67 |
+
return local
|
| 68 |
+
from huggingface_hub import hf_hub_download
|
| 69 |
+
return hf_hub_download(repo_id=model_id, filename=filename, **hub_kwargs)
|
src/tensorpeft/cp.py
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 Angshul Majumdar. Licensed under the Apache License, Version 2.0.
|
| 2 |
+
"""Rank-one CP approximation by higher-order power iteration (HOPM)."""
|
| 3 |
+
import torch
|
| 4 |
+
|
| 5 |
+
_L = "abcdefgh"
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def contract_except(T, us, n):
|
| 9 |
+
N = T.dim()
|
| 10 |
+
idx = _L[:N]
|
| 11 |
+
spec = idx + "," + ",".join(idx[m] for m in range(N) if m != n) + "->" + idx[n]
|
| 12 |
+
return torch.einsum(spec, T, *[us[m] for m in range(N) if m != n])
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def cp_value(T, us):
|
| 16 |
+
idx = _L[: T.dim()]
|
| 17 |
+
return torch.einsum(idx + "," + ",".join(idx) + "->", T, *us)
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def outer(us):
|
| 21 |
+
out = us[0]
|
| 22 |
+
for u in us[1:]:
|
| 23 |
+
out = out[..., None] * u
|
| 24 |
+
return out
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
@torch.no_grad()
|
| 28 |
+
def rank1_cp(T, iters=30, tol=1e-6):
|
| 29 |
+
"""Unit factors u_1..u_N (approximately) maximising sigma = <T, u_1 o ... o u_N>.
|
| 30 |
+
|
| 31 |
+
T has shape (leading..., d_in, d_out) with one or two leading modes. Initialised from the top
|
| 32 |
+
singular pair of the largest (d_in x d_out) slice (deterministic), then HOPM sweeps."""
|
| 33 |
+
N = T.dim()
|
| 34 |
+
lead_shape = T.shape[:-2]
|
| 35 |
+
slices = T.reshape(-1, T.shape[-2], T.shape[-1])
|
| 36 |
+
j = int(slices.flatten(1).norm(dim=1).argmax())
|
| 37 |
+
M = slices[j]
|
| 38 |
+
small = M @ M.T if M.shape[0] <= M.shape[1] else M.T @ M # exact top pair via Gram eigvec
|
| 39 |
+
_, vecs = torch.linalg.eigh(small.double())
|
| 40 |
+
v = vecs[:, -1].to(T.dtype)
|
| 41 |
+
if M.shape[0] <= M.shape[1]:
|
| 42 |
+
u_in, u_out = v, M.T @ v
|
| 43 |
+
else:
|
| 44 |
+
u_in, u_out = M @ v, v
|
| 45 |
+
u_in, u_out = u_in / u_in.norm().clamp_min(1e-30), u_out / u_out.norm().clamp_min(1e-30)
|
| 46 |
+
core = torch.einsum("sij,i,j->s", slices, u_in, u_out).reshape(lead_shape)
|
| 47 |
+
if core.dim() == 1:
|
| 48 |
+
us = [core / core.norm().clamp_min(1e-30)]
|
| 49 |
+
else:
|
| 50 |
+
Uc, _, Vc = torch.linalg.svd(core, full_matrices=False)
|
| 51 |
+
us = [Uc[:, 0], Vc[0]]
|
| 52 |
+
us = us + [u_in, u_out]
|
| 53 |
+
sigma = cp_value(T, us)
|
| 54 |
+
for _ in range(iters):
|
| 55 |
+
for n in range(N):
|
| 56 |
+
w = contract_except(T, us, n)
|
| 57 |
+
us[n] = w / w.norm().clamp_min(1e-30)
|
| 58 |
+
new = cp_value(T, us)
|
| 59 |
+
done = abs(float(new) - float(sigma)) <= tol * max(abs(float(new)), 1e-30)
|
| 60 |
+
sigma = new
|
| 61 |
+
if done:
|
| 62 |
+
break
|
| 63 |
+
if sigma < 0:
|
| 64 |
+
us[-1] = -us[-1]
|
| 65 |
+
sigma = -sigma
|
| 66 |
+
return us, float(sigma)
|
src/tensorpeft/layers.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 Angshul Majumdar. Licensed under the Apache License, Version 2.0.
|
| 2 |
+
"""Discovery of adaptable weight matrices ('slots') and the forward hooks that apply dW."""
|
| 3 |
+
import re
|
| 4 |
+
from collections import defaultdict
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
import torch.nn as nn
|
| 8 |
+
|
| 9 |
+
try:
|
| 10 |
+
from transformers.pytorch_utils import Conv1D
|
| 11 |
+
except Exception: # pragma: no cover
|
| 12 |
+
class Conv1D(nn.Module):
|
| 13 |
+
pass
|
| 14 |
+
|
| 15 |
+
# GPT-2 family: Conv1D c_attn holds [q | k | v] as three equal column blocks
|
| 16 |
+
FUSED_EQUAL_SPLIT = {"c_attn": ("q", "k", "v")}
|
| 17 |
+
_LAYER_RE = re.compile(r"\.(\d+)\.")
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
class Slot:
|
| 21 |
+
"""One adapted matrix, always viewed as (d_in, d_out) so that the adapter adds x @ dW."""
|
| 22 |
+
|
| 23 |
+
def __init__(self, module_name, module, layer, role, c0, c1, d_in):
|
| 24 |
+
self.module_name, self.module = module_name, module
|
| 25 |
+
self.layer, self.role = layer, role
|
| 26 |
+
self.c0, self.c1, self.d_in = c0, c1, d_in
|
| 27 |
+
self.conv1d = isinstance(module, Conv1D)
|
| 28 |
+
|
| 29 |
+
@property
|
| 30 |
+
def shape(self):
|
| 31 |
+
return (self.d_in, self.c1 - self.c0)
|
| 32 |
+
|
| 33 |
+
def _view(self, w):
|
| 34 |
+
return w[:, self.c0:self.c1] if self.conv1d else w[self.c0:self.c1, :].T
|
| 35 |
+
|
| 36 |
+
def weight(self):
|
| 37 |
+
return self._view(self.module.weight)
|
| 38 |
+
|
| 39 |
+
def grad(self):
|
| 40 |
+
return self._view(self.module.weight.grad)
|
| 41 |
+
|
| 42 |
+
def __repr__(self):
|
| 43 |
+
return f"Slot(layer {self.layer}: {self.role} {list(self.shape)})"
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def find_slots(model, target_modules="all-linear", exclude_modules=(), split_fused_qkv=True):
|
| 47 |
+
slots = []
|
| 48 |
+
for name, mod in model.named_modules():
|
| 49 |
+
if not isinstance(mod, (nn.Linear, Conv1D)):
|
| 50 |
+
continue
|
| 51 |
+
if any(e in name for e in exclude_modules):
|
| 52 |
+
continue
|
| 53 |
+
m = _LAYER_RE.search(name)
|
| 54 |
+
if m is None:
|
| 55 |
+
continue
|
| 56 |
+
role = name[m.end():]
|
| 57 |
+
w = mod.weight
|
| 58 |
+
d_in, d_out = (w.shape[0], w.shape[1]) if isinstance(mod, Conv1D) else (w.shape[1], w.shape[0])
|
| 59 |
+
parts = FUSED_EQUAL_SPLIT.get(role.split(".")[-1]) if (split_fused_qkv and isinstance(mod, Conv1D)) else None
|
| 60 |
+
if parts and d_out % len(parts) == 0:
|
| 61 |
+
wd = d_out // len(parts)
|
| 62 |
+
for j, p in enumerate(parts):
|
| 63 |
+
slots.append(Slot(name, mod, int(m.group(1)), f"{role}.{p}", j * wd, (j + 1) * wd, d_in))
|
| 64 |
+
else:
|
| 65 |
+
slots.append(Slot(name, mod, int(m.group(1)), role, 0, d_out, d_in))
|
| 66 |
+
if target_modules != "all-linear":
|
| 67 |
+
if isinstance(target_modules, str):
|
| 68 |
+
target_modules = [target_modules]
|
| 69 |
+
pats = [re.compile(r"(^|\.)" + re.escape(t) + r"$") for t in target_modules]
|
| 70 |
+
slots = [s for s in slots if any(p.search(s.role) for p in pats)]
|
| 71 |
+
if not slots:
|
| 72 |
+
raise ValueError(f"TensorPEFT: no target modules matched {target_modules!r}")
|
| 73 |
+
return slots
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def attach_hooks(slots, delta_fn):
|
| 77 |
+
"""Forward hooks adding delta_fn(slot, x) to each slot's output columns (cast to the output dtype)."""
|
| 78 |
+
by_mod = defaultdict(list)
|
| 79 |
+
for s in slots:
|
| 80 |
+
by_mod[id(s.module)].append(s)
|
| 81 |
+
handles = []
|
| 82 |
+
for ss in by_mod.values():
|
| 83 |
+
ss = sorted(ss, key=lambda s: s.c0)
|
| 84 |
+
mod = ss[0].module
|
| 85 |
+
width = mod.weight.shape[1] if ss[0].conv1d else mod.weight.shape[0]
|
| 86 |
+
|
| 87 |
+
def hook(module, inputs, out, ss=ss, width=width):
|
| 88 |
+
x = inputs[0]
|
| 89 |
+
deltas = [delta_fn(s, x) for s in ss]
|
| 90 |
+
if all(d is None for d in deltas):
|
| 91 |
+
return out
|
| 92 |
+
pieces, pos = [], 0
|
| 93 |
+
for s, d in zip(ss, deltas):
|
| 94 |
+
if s.c0 > pos:
|
| 95 |
+
pieces.append(out.new_zeros(*out.shape[:-1], s.c0 - pos))
|
| 96 |
+
pieces.append(out.new_zeros(*out.shape[:-1], s.c1 - s.c0) if d is None else d.to(out.dtype))
|
| 97 |
+
pos = s.c1
|
| 98 |
+
if pos < width:
|
| 99 |
+
pieces.append(out.new_zeros(*out.shape[:-1], width - pos))
|
| 100 |
+
return out + (pieces[0] if len(pieces) == 1 else torch.cat(pieces, dim=-1))
|
| 101 |
+
|
| 102 |
+
handles.append(mod.register_forward_hook(hook))
|
| 103 |
+
return handles
|
src/tensorpeft/model.py
ADDED
|
@@ -0,0 +1,307 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 Angshul Majumdar. Licensed under the Apache License, Version 2.0.
|
| 2 |
+
"""TensorPEFT: tensor gradient-pursuit adapters.
|
| 3 |
+
|
| 4 |
+
Matrices of equal shape across all transformer blocks are stacked into one update tensor
|
| 5 |
+
(layer x [role] x d_in x d_out). Each group's update is a sum of rank-one CP atoms
|
| 6 |
+
|
| 7 |
+
dW = sum_i a_i u_i(layer) o u_i(role) o u_i(in) o u_i(out).
|
| 8 |
+
|
| 9 |
+
``init_atoms`` runs one gradient probe, and the atoms are the leading rank-one CP components of
|
| 10 |
+
the negative gradient tensor (HOPM with deflation). Coefficients start at zero, so the model is
|
| 11 |
+
unchanged until training starts; all coefficients and factors are then trained normally.
|
| 12 |
+
"""
|
| 13 |
+
import functools
|
| 14 |
+
import os
|
| 15 |
+
|
| 16 |
+
import torch
|
| 17 |
+
import torch.nn as nn
|
| 18 |
+
|
| 19 |
+
from . import __version__
|
| 20 |
+
from .config import CONFIG_NAME, WEIGHTS_NAME, TensorPEFTConfig, _resolve
|
| 21 |
+
from .cp import cp_value, outer, rank1_cp
|
| 22 |
+
from .layers import attach_hooks, find_slots
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class TensorGroup(nn.Module):
|
| 26 |
+
def __init__(self, slots, k, alpha_scale):
|
| 27 |
+
super().__init__()
|
| 28 |
+
self.slots = slots
|
| 29 |
+
self.d_in, self.d_out = slots[0].shape
|
| 30 |
+
self.layers = sorted({s.layer for s in slots})
|
| 31 |
+
self.roles = sorted({s.role for s in slots})
|
| 32 |
+
self.has_p = len(self.roles) > 1
|
| 33 |
+
self.shape = (len(self.layers),) + ((len(self.roles),) if self.has_p else ()) + (self.d_in, self.d_out)
|
| 34 |
+
self.index = {id(s): (self.layers.index(s.layer),) + ((self.roles.index(s.role),) if self.has_p else ())
|
| 35 |
+
for s in slots}
|
| 36 |
+
self.k = k
|
| 37 |
+
self.alpha_scale = alpha_scale
|
| 38 |
+
dev = slots[0].module.weight.device
|
| 39 |
+
self.alpha_raw = nn.Parameter(torch.zeros(k, device=dev))
|
| 40 |
+
self.factors = nn.ParameterList([nn.Parameter(torch.zeros(k, n, device=dev)) for n in self.shape])
|
| 41 |
+
self.register_buffer("initialized", torch.tensor(False, device=dev))
|
| 42 |
+
self._cache = None
|
| 43 |
+
|
| 44 |
+
@property
|
| 45 |
+
def name(self):
|
| 46 |
+
return f"{'+'.join(self.roles)} {list(self.shape)}"
|
| 47 |
+
|
| 48 |
+
def n_params(self):
|
| 49 |
+
return self.k * (1 + sum(self.shape))
|
| 50 |
+
|
| 51 |
+
def meta(self):
|
| 52 |
+
return {"roles": self.roles, "layers": self.layers, "shape": list(self.shape)}
|
| 53 |
+
|
| 54 |
+
def prepare(self):
|
| 55 |
+
if not bool(self.initialized):
|
| 56 |
+
self._cache = None
|
| 57 |
+
return
|
| 58 |
+
a = self.alpha_raw * self.alpha_scale
|
| 59 |
+
F = list(self.factors)
|
| 60 |
+
C = torch.einsum("k,kl,kp->klp", a, F[0], F[1]) if self.has_p else a[:, None] * F[0]
|
| 61 |
+
self._cache = (C, F[-2], F[-1])
|
| 62 |
+
|
| 63 |
+
def delta(self, slot, x):
|
| 64 |
+
if self._cache is None:
|
| 65 |
+
return None
|
| 66 |
+
C, U_in, U_out = self._cache
|
| 67 |
+
c = C[(slice(None),) + self.index[id(slot)]]
|
| 68 |
+
return ((x.to(U_in.dtype) @ U_in.T) * c) @ U_out
|
| 69 |
+
|
| 70 |
+
@torch.no_grad()
|
| 71 |
+
def dense_delta(self, slot):
|
| 72 |
+
self.prepare()
|
| 73 |
+
if self._cache is None:
|
| 74 |
+
return None
|
| 75 |
+
C, U_in, U_out = self._cache
|
| 76 |
+
c = C[(slice(None),) + self.index[id(slot)]]
|
| 77 |
+
return (U_in.T * c) @ U_out
|
| 78 |
+
|
| 79 |
+
@torch.no_grad()
|
| 80 |
+
def stacked_grad(self):
|
| 81 |
+
w = self.slots[0].module.weight
|
| 82 |
+
G = torch.zeros(self.shape, dtype=torch.float32, device=w.device)
|
| 83 |
+
for s in self.slots:
|
| 84 |
+
G[self.index[id(s)]] = s.grad().float()
|
| 85 |
+
return G
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
class TensorPEFTModel(nn.Module):
|
| 89 |
+
"""Wraps a Hugging Face causal LM. Use :func:`get_tensorpeft_model` to create one."""
|
| 90 |
+
|
| 91 |
+
def __init__(self, base_model, config: TensorPEFTConfig):
|
| 92 |
+
super().__init__()
|
| 93 |
+
self.base_model = base_model
|
| 94 |
+
self.peft_config = config
|
| 95 |
+
config.tensorpeft_version = __version__
|
| 96 |
+
if config.base_model_name_or_path is None:
|
| 97 |
+
config.base_model_name_or_path = getattr(getattr(base_model, "config", None), "_name_or_path", None)
|
| 98 |
+
for p in base_model.parameters():
|
| 99 |
+
p.requires_grad_(False)
|
| 100 |
+
self.slots = find_slots(base_model, config.target_modules, config.exclude_modules, config.split_fused_qkv)
|
| 101 |
+
by_shape = {}
|
| 102 |
+
for s in self.slots:
|
| 103 |
+
by_shape.setdefault(s.shape, []).append(s)
|
| 104 |
+
self.groups = nn.ModuleList([TensorGroup(ss, config.atoms_per_group, config.alpha_scale)
|
| 105 |
+
for ss in by_shape.values()])
|
| 106 |
+
self._group_of = {id(s): g for g in self.groups for s in g.slots}
|
| 107 |
+
self._handles = attach_hooks(self.slots, lambda s, x: self._group_of[id(s)].delta(s, x))
|
| 108 |
+
self._handles.append(base_model.register_forward_pre_hook(lambda m, a: self._prepare()))
|
| 109 |
+
self._modules_w = list({id(s.module): s.module for s in self.slots}.values())
|
| 110 |
+
|
| 111 |
+
# forward with the base model's signature (so HF Trainer keeps the right dataset columns)
|
| 112 |
+
def forward(*args, **kwargs):
|
| 113 |
+
return self.base_model(*args, **kwargs)
|
| 114 |
+
functools.update_wrapper(forward, base_model.forward)
|
| 115 |
+
self.forward = forward
|
| 116 |
+
|
| 117 |
+
# ------------------------------------------------------------------ plumbing
|
| 118 |
+
def _prepare(self):
|
| 119 |
+
for g in self.groups:
|
| 120 |
+
g.prepare()
|
| 121 |
+
|
| 122 |
+
def __getattr__(self, name):
|
| 123 |
+
try:
|
| 124 |
+
return super().__getattr__(name)
|
| 125 |
+
except AttributeError:
|
| 126 |
+
return getattr(super().__getattr__("base_model"), name)
|
| 127 |
+
|
| 128 |
+
def get_base_model(self):
|
| 129 |
+
return self.base_model
|
| 130 |
+
|
| 131 |
+
@property
|
| 132 |
+
def is_initialized(self):
|
| 133 |
+
return all(bool(g.initialized) for g in self.groups)
|
| 134 |
+
|
| 135 |
+
def get_nb_trainable_parameters(self):
|
| 136 |
+
trainable = sum(p.numel() for p in self.parameters() if p.requires_grad)
|
| 137 |
+
total = sum(p.numel() for p in self.base_model.parameters()) + trainable
|
| 138 |
+
return trainable, total
|
| 139 |
+
|
| 140 |
+
def print_trainable_parameters(self):
|
| 141 |
+
t, a = self.get_nb_trainable_parameters()
|
| 142 |
+
print(f"trainable params: {t:,} || all params: {a:,} || trainable%: {100 * t / a:.4f}")
|
| 143 |
+
|
| 144 |
+
def describe(self):
|
| 145 |
+
return [f"{g.name}: {g.k} atoms, {g.n_params():,} params" for g in self.groups]
|
| 146 |
+
|
| 147 |
+
# ------------------------------------------------------------------ initialisation
|
| 148 |
+
def init_atoms(self, batches, n_batches=None, loss_fn=None, verbose=True):
|
| 149 |
+
"""Choose all atoms from one gradient probe.
|
| 150 |
+
|
| 151 |
+
batches: iterable of dicts (``input_ids``, optional ``attention_mask`` / ``labels``) or of
|
| 152 |
+
input-id tensors (causal-LM labels = inputs). A DataLoader works.
|
| 153 |
+
n_batches: how many batches to use (default ``config.init_batches``).
|
| 154 |
+
loss_fn: optional ``loss_fn(base_model, batch) -> scalar``.
|
| 155 |
+
Returns, per group, each atom's initial descent rate <-G, atom> (= -dL/da at a = 0).
|
| 156 |
+
"""
|
| 157 |
+
n_batches = n_batches or self.peft_config.init_batches
|
| 158 |
+
dev = self._modules_w[0].weight.device
|
| 159 |
+
was_training = self.base_model.training
|
| 160 |
+
self.base_model.eval()
|
| 161 |
+
for m in self._modules_w:
|
| 162 |
+
m.weight.requires_grad_(True)
|
| 163 |
+
m.weight.grad = None
|
| 164 |
+
used = 0
|
| 165 |
+
for b in batches:
|
| 166 |
+
if used >= n_batches:
|
| 167 |
+
break
|
| 168 |
+
if loss_fn is not None:
|
| 169 |
+
loss = loss_fn(self.base_model, b)
|
| 170 |
+
else:
|
| 171 |
+
if torch.is_tensor(b):
|
| 172 |
+
b = {"input_ids": b}
|
| 173 |
+
b = {k: (v.to(dev) if torch.is_tensor(v) else v) for k, v in b.items()}
|
| 174 |
+
if "labels" not in b:
|
| 175 |
+
b["labels"] = b["input_ids"]
|
| 176 |
+
loss = self.base_model(**b).loss
|
| 177 |
+
loss.backward()
|
| 178 |
+
used += 1
|
| 179 |
+
if used == 0:
|
| 180 |
+
raise ValueError("init_atoms: no batches given")
|
| 181 |
+
report = []
|
| 182 |
+
with torch.no_grad():
|
| 183 |
+
for g in self.groups:
|
| 184 |
+
negG = -g.stacked_grad() / used # mean gradient over the batches used
|
| 185 |
+
T = negG.clone()
|
| 186 |
+
sig = []
|
| 187 |
+
for i in range(g.k):
|
| 188 |
+
us, s = rank1_cp(T, iters=self.peft_config.hopm_iters)
|
| 189 |
+
for pl, u in zip(g.factors, us):
|
| 190 |
+
pl[i].copy_(u)
|
| 191 |
+
T = T - s * outer(us) # deflation
|
| 192 |
+
sig.append(float(cp_value(negG, us)))
|
| 193 |
+
del negG, T
|
| 194 |
+
g.alpha_raw.zero_()
|
| 195 |
+
g.initialized.fill_(True)
|
| 196 |
+
report.append(sig)
|
| 197 |
+
for m in self._modules_w:
|
| 198 |
+
m.weight.grad = None
|
| 199 |
+
m.weight.requires_grad_(False)
|
| 200 |
+
for p in self.parameters():
|
| 201 |
+
p.grad = None
|
| 202 |
+
self.base_model.train(was_training)
|
| 203 |
+
if verbose:
|
| 204 |
+
for g, s in zip(self.groups, report):
|
| 205 |
+
print(f"TensorPEFT {g.name}: {g.k} atoms, initial descent rate {s[0]:.4g} .. {s[-1]:.4g}")
|
| 206 |
+
return report
|
| 207 |
+
|
| 208 |
+
# ------------------------------------------------------------------ save / load
|
| 209 |
+
def adapter_state_dict(self):
|
| 210 |
+
sd = {}
|
| 211 |
+
for gi, g in enumerate(self.groups):
|
| 212 |
+
sd[f"group{gi}.alpha_raw"] = g.alpha_raw.detach().cpu().contiguous()
|
| 213 |
+
for m, pl in enumerate(g.factors):
|
| 214 |
+
sd[f"group{gi}.factor{m}"] = pl.detach().cpu().contiguous()
|
| 215 |
+
return sd
|
| 216 |
+
|
| 217 |
+
def save_pretrained(self, save_directory, **kwargs):
|
| 218 |
+
if not self.is_initialized:
|
| 219 |
+
raise RuntimeError("call init_atoms(...) before saving")
|
| 220 |
+
from safetensors.torch import save_file
|
| 221 |
+
os.makedirs(save_directory, exist_ok=True)
|
| 222 |
+
self.peft_config.save_pretrained(save_directory, extra={"groups": [g.meta() for g in self.groups]})
|
| 223 |
+
save_file(self.adapter_state_dict(), os.path.join(save_directory, WEIGHTS_NAME),
|
| 224 |
+
metadata={"format": "pt", "library": "tensorpeft"})
|
| 225 |
+
readme = os.path.join(save_directory, "README.md")
|
| 226 |
+
if not os.path.exists(readme):
|
| 227 |
+
with open(readme, "w") as f:
|
| 228 |
+
f.write(_model_card(self))
|
| 229 |
+
|
| 230 |
+
@classmethod
|
| 231 |
+
def from_pretrained(cls, base_model, model_id, is_trainable=False, **hub_kwargs):
|
| 232 |
+
config, raw = TensorPEFTConfig.from_pretrained(model_id, **hub_kwargs)
|
| 233 |
+
model = cls(base_model, config)
|
| 234 |
+
saved = raw.get("groups")
|
| 235 |
+
if saved is not None:
|
| 236 |
+
mine = [g.meta() for g in model.groups]
|
| 237 |
+
if [(m["roles"], m["shape"]) for m in mine] != [(m["roles"], m["shape"]) for m in saved]:
|
| 238 |
+
raise ValueError("adapter does not match this base model's architecture")
|
| 239 |
+
from safetensors.torch import load_file
|
| 240 |
+
sd = load_file(_resolve(model_id, WEIGHTS_NAME, **hub_kwargs))
|
| 241 |
+
with torch.no_grad():
|
| 242 |
+
for gi, g in enumerate(model.groups):
|
| 243 |
+
g.alpha_raw.copy_(sd[f"group{gi}.alpha_raw"])
|
| 244 |
+
for m, pl in enumerate(g.factors):
|
| 245 |
+
pl.copy_(sd[f"group{gi}.factor{m}"])
|
| 246 |
+
g.initialized.fill_(True)
|
| 247 |
+
for p in model.parameters():
|
| 248 |
+
if p.requires_grad and not is_trainable:
|
| 249 |
+
p.requires_grad_(False)
|
| 250 |
+
return model
|
| 251 |
+
|
| 252 |
+
def push_to_hub(self, repo_id, private=None, token=None, commit_message="Upload TensorPEFT adapter"):
|
| 253 |
+
import tempfile
|
| 254 |
+
from huggingface_hub import HfApi
|
| 255 |
+
api = HfApi(token=token)
|
| 256 |
+
api.create_repo(repo_id, private=private, exist_ok=True)
|
| 257 |
+
with tempfile.TemporaryDirectory() as d:
|
| 258 |
+
self.save_pretrained(d)
|
| 259 |
+
return api.upload_folder(repo_id=repo_id, folder_path=d, commit_message=commit_message)
|
| 260 |
+
|
| 261 |
+
# ------------------------------------------------------------------ merge
|
| 262 |
+
@torch.no_grad()
|
| 263 |
+
def merge_and_unload(self):
|
| 264 |
+
"""Write dW into the base weights and return the plain base model (no inference overhead)."""
|
| 265 |
+
for s in self.slots:
|
| 266 |
+
d = self._group_of[id(s)].dense_delta(s)
|
| 267 |
+
if d is not None:
|
| 268 |
+
w = s.weight()
|
| 269 |
+
w.add_(d.to(w.dtype))
|
| 270 |
+
return self.unload()
|
| 271 |
+
|
| 272 |
+
def unload(self):
|
| 273 |
+
"""Remove the adapter hooks without merging and return the base model."""
|
| 274 |
+
for h in self._handles:
|
| 275 |
+
h.remove()
|
| 276 |
+
self._handles = []
|
| 277 |
+
return self.base_model
|
| 278 |
+
|
| 279 |
+
|
| 280 |
+
def get_tensorpeft_model(model, config: TensorPEFTConfig = None):
|
| 281 |
+
"""Wrap a Hugging Face model with TensorPEFT. Call ``.init_atoms(batches)`` before training."""
|
| 282 |
+
return TensorPEFTModel(model, config or TensorPEFTConfig())
|
| 283 |
+
|
| 284 |
+
|
| 285 |
+
def _model_card(m):
|
| 286 |
+
c = m.peft_config
|
| 287 |
+
t, _ = m.get_nb_trainable_parameters()
|
| 288 |
+
return f"""---
|
| 289 |
+
library_name: tensorpeft
|
| 290 |
+
base_model: {c.base_model_name_or_path or 'unknown'}
|
| 291 |
+
tags:
|
| 292 |
+
- tensorpeft
|
| 293 |
+
- peft
|
| 294 |
+
---
|
| 295 |
+
|
| 296 |
+
# TensorPEFT adapter
|
| 297 |
+
|
| 298 |
+
Base model: `{c.base_model_name_or_path}`; {c.atoms_per_group} atoms per tensor group; {t:,} adapter parameters.
|
| 299 |
+
|
| 300 |
+
```python
|
| 301 |
+
from transformers import AutoModelForCausalLM
|
| 302 |
+
from tensorpeft import TensorPEFTModel
|
| 303 |
+
base = AutoModelForCausalLM.from_pretrained("{c.base_model_name_or_path}")
|
| 304 |
+
model = TensorPEFTModel.from_pretrained(base, "<this repo id>")
|
| 305 |
+
model = model.merge_and_unload() # optional
|
| 306 |
+
```
|
| 307 |
+
"""
|
src/tensorpeft/trainer.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 Angshul Majumdar. Licensed under the Apache License, Version 2.0.
|
| 2 |
+
"""Hugging Face Trainer that saves TensorPEFT checkpoints as adapters (not the full model)."""
|
| 3 |
+
import os
|
| 4 |
+
|
| 5 |
+
from transformers import Trainer
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class TensorPEFTTrainer(Trainer):
|
| 9 |
+
"""Drop-in ``transformers.Trainer``; checkpoints and ``save_model`` write only the adapter."""
|
| 10 |
+
|
| 11 |
+
def _save(self, output_dir=None, state_dict=None):
|
| 12 |
+
output_dir = output_dir if output_dir is not None else self.args.output_dir
|
| 13 |
+
os.makedirs(output_dir, exist_ok=True)
|
| 14 |
+
model = self.accelerator.unwrap_model(self.model)
|
| 15 |
+
if self.args.should_save:
|
| 16 |
+
model.save_pretrained(output_dir)
|
| 17 |
+
if self.processing_class is not None:
|
| 18 |
+
self.processing_class.save_pretrained(output_dir)
|
tests/test_tensorpeft.py
ADDED
|
@@ -0,0 +1,216 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""pytest tests for TensorPEFT (CPU, tiny random models)."""
|
| 2 |
+
import math
|
| 3 |
+
|
| 4 |
+
import pytest
|
| 5 |
+
import torch
|
| 6 |
+
import torch.nn.functional as F
|
| 7 |
+
import transformers as T
|
| 8 |
+
|
| 9 |
+
from tensorpeft import TensorPEFTConfig, TensorPEFTModel, get_tensorpeft_model
|
| 10 |
+
from tensorpeft.cp import cp_value, contract_except, outer, rank1_cp
|
| 11 |
+
|
| 12 |
+
V = 97
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def tiny(arch):
|
| 16 |
+
common = dict(vocab_size=V, max_position_embeddings=64)
|
| 17 |
+
if arch == "gpt2":
|
| 18 |
+
return T.GPT2LMHeadModel(T.GPT2Config(n_layer=3, n_embd=32, n_head=4, vocab_size=V, n_positions=64))
|
| 19 |
+
if arch == "llama":
|
| 20 |
+
return T.LlamaForCausalLM(T.LlamaConfig(num_hidden_layers=3, hidden_size=32, intermediate_size=48,
|
| 21 |
+
num_attention_heads=4, num_key_value_heads=2, **common))
|
| 22 |
+
if arch == "qwen2":
|
| 23 |
+
return T.Qwen2ForCausalLM(T.Qwen2Config(num_hidden_layers=3, hidden_size=32, intermediate_size=48,
|
| 24 |
+
num_attention_heads=4, num_key_value_heads=2, **common))
|
| 25 |
+
if arch == "mistral":
|
| 26 |
+
return T.MistralForCausalLM(T.MistralConfig(num_hidden_layers=3, hidden_size=32, intermediate_size=48,
|
| 27 |
+
num_attention_heads=4, num_key_value_heads=2, **common))
|
| 28 |
+
if arch == "phi":
|
| 29 |
+
return T.PhiForCausalLM(T.PhiConfig(num_hidden_layers=3, hidden_size=32, intermediate_size=48,
|
| 30 |
+
num_attention_heads=4, **common))
|
| 31 |
+
if arch == "gemma":
|
| 32 |
+
return T.GemmaForCausalLM(T.GemmaConfig(num_hidden_layers=3, hidden_size=32, intermediate_size=48,
|
| 33 |
+
num_attention_heads=4, num_key_value_heads=2, head_dim=8, **common))
|
| 34 |
+
raise ValueError(arch)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
ARCHS = ["gpt2", "llama", "qwen2", "mistral", "phi", "gemma"]
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def batches(n=2, b=3, t=17, seed=0):
|
| 41 |
+
g = torch.Generator().manual_seed(seed)
|
| 42 |
+
return [torch.randint(0, V, (b, t), generator=g) for _ in range(n)]
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def lm_loss(model, x):
|
| 46 |
+
logits = model(x[:, :-1]).logits
|
| 47 |
+
return F.cross_entropy(logits.float().flatten(0, 1), x[:, 1:].flatten())
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
@pytest.fixture(autouse=True)
|
| 51 |
+
def _seed():
|
| 52 |
+
torch.manual_seed(0)
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
# ---------------------------------------------------------------- CP / HOPM
|
| 56 |
+
@pytest.mark.parametrize("shape", [(5, 30, 40), (6, 4, 30, 20)])
|
| 57 |
+
def test_hopm_recovers_planted_rank1(shape):
|
| 58 |
+
us = [F.normalize(torch.randn(n, dtype=torch.float64), dim=0) for n in shape]
|
| 59 |
+
T_ = 5.0 * outer(us) + 0.05 * torch.randn(shape, dtype=torch.float64)
|
| 60 |
+
est, sig = rank1_cp(T_, iters=50)
|
| 61 |
+
assert min(abs(float(a @ b)) for a, b in zip(us, est)) > 0.99
|
| 62 |
+
assert abs(sig - 5.0) < 0.3
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def test_hopm_fixed_point():
|
| 66 |
+
T_ = torch.randn(4, 3, 20, 25, dtype=torch.float64)
|
| 67 |
+
est, sig = rank1_cp(T_, iters=500, tol=1e-14)
|
| 68 |
+
res = max(float((contract_except(T_, est, n) / sig - est[n]).abs().max()) for n in range(4))
|
| 69 |
+
assert res < 1e-6
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
# ---------------------------------------------------------------- model wrapping
|
| 73 |
+
@pytest.mark.parametrize("arch", ARCHS)
|
| 74 |
+
def test_unchanged_before_and_after_init(arch):
|
| 75 |
+
base = tiny(arch).eval()
|
| 76 |
+
x = batches(1)[0]
|
| 77 |
+
ref = base(x).logits.detach().clone()
|
| 78 |
+
model = get_tensorpeft_model(base, TensorPEFTConfig(atoms_per_group=2))
|
| 79 |
+
assert torch.equal(model(x).logits, ref)
|
| 80 |
+
model.init_atoms(batches(2), verbose=False)
|
| 81 |
+
assert model.is_initialized
|
| 82 |
+
assert float((model(x).logits - ref).abs().max()) < 1e-6 # a = 0 at init
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
@pytest.mark.parametrize("arch", ARCHS)
|
| 86 |
+
def test_initial_gradient_is_steepest_cp_direction(arch):
|
| 87 |
+
"""dL/d alpha_raw_i at init = -alpha_scale * sigma_i (atoms are descent directions)."""
|
| 88 |
+
torch.set_default_dtype(torch.float64)
|
| 89 |
+
try:
|
| 90 |
+
base = tiny(arch).eval()
|
| 91 |
+
model = get_tensorpeft_model(base, TensorPEFTConfig(atoms_per_group=3))
|
| 92 |
+
xs = batches(2)
|
| 93 |
+
sig = model.init_atoms(xs, verbose=False)
|
| 94 |
+
loss = sum(lm_loss(model, x) for x in xs) / len(xs)
|
| 95 |
+
grads = torch.autograd.grad(loss, [g.alpha_raw for g in model.groups])
|
| 96 |
+
for g, gr, s in zip(model.groups, grads, sig):
|
| 97 |
+
expect = -g.alpha_scale * torch.tensor(s, dtype=gr.dtype)
|
| 98 |
+
assert torch.allclose(gr, expect, rtol=1e-6, atol=1e-10), (arch, gr, expect)
|
| 99 |
+
finally:
|
| 100 |
+
torch.set_default_dtype(torch.float32)
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
@pytest.mark.parametrize("arch", ARCHS)
|
| 104 |
+
def test_merge_equals_hooks_and_save_load_roundtrip(arch, tmp_path):
|
| 105 |
+
torch.set_default_dtype(torch.float64)
|
| 106 |
+
try:
|
| 107 |
+
base = tiny(arch).eval()
|
| 108 |
+
state = {k: v.clone() for k, v in base.state_dict().items()}
|
| 109 |
+
x = batches(1, seed=5)[0]
|
| 110 |
+
ref = base(x).logits.detach().clone()
|
| 111 |
+
model = get_tensorpeft_model(base, TensorPEFTConfig(atoms_per_group=2))
|
| 112 |
+
model.init_atoms(batches(2), verbose=False)
|
| 113 |
+
with torch.no_grad():
|
| 114 |
+
for p in model.parameters():
|
| 115 |
+
if p.requires_grad:
|
| 116 |
+
p.add_(0.3 * torch.randn_like(p))
|
| 117 |
+
hooked = model(x).logits.detach().clone()
|
| 118 |
+
assert float((hooked - ref).abs().max()) > 1e-3 # the adapter does change the model
|
| 119 |
+
model.save_pretrained(tmp_path)
|
| 120 |
+
# reload onto a fresh copy of the same base
|
| 121 |
+
base2 = tiny(arch).eval()
|
| 122 |
+
base2.load_state_dict(state)
|
| 123 |
+
re = TensorPEFTModel.from_pretrained(base2, str(tmp_path))
|
| 124 |
+
assert float((re(x).logits - hooked).abs().max()) < 1e-10
|
| 125 |
+
merged = model.merge_and_unload()
|
| 126 |
+
assert float((merged(x).logits - hooked).abs().max()) < 1e-10
|
| 127 |
+
finally:
|
| 128 |
+
torch.set_default_dtype(torch.float32)
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def test_param_count_and_grouping_gpt2():
|
| 132 |
+
base = tiny("gpt2")
|
| 133 |
+
m = get_tensorpeft_model(base, TensorPEFTConfig(atoms_per_group=4))
|
| 134 |
+
names = [g.name for g in m.groups]
|
| 135 |
+
assert names[0].startswith("attn.c_attn.k+attn.c_attn.q+attn.c_attn.v+attn.c_proj [3, 4, 32, 32]")
|
| 136 |
+
expect = 4 * ((1 + 3 + 4 + 32 + 32) + (1 + 3 + 32 + 128) + (1 + 3 + 128 + 32))
|
| 137 |
+
assert m.get_nb_trainable_parameters()[0] == expect
|
| 138 |
+
assert all(not p.requires_grad for p in base.parameters())
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def test_target_modules_subset():
|
| 142 |
+
base = tiny("llama")
|
| 143 |
+
m = get_tensorpeft_model(base, TensorPEFTConfig(target_modules=["q_proj", "v_proj"], atoms_per_group=1))
|
| 144 |
+
roles = sorted({s.role for s in m.slots})
|
| 145 |
+
assert roles == ["self_attn.q_proj", "self_attn.v_proj"]
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def test_wrong_architecture_rejected(tmp_path):
|
| 149 |
+
m = get_tensorpeft_model(tiny("llama"), TensorPEFTConfig(atoms_per_group=1))
|
| 150 |
+
m.init_atoms(batches(1), verbose=False)
|
| 151 |
+
m.save_pretrained(tmp_path)
|
| 152 |
+
with pytest.raises(ValueError):
|
| 153 |
+
TensorPEFTModel.from_pretrained(tiny("gpt2"), str(tmp_path))
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def test_dict_batches_with_attention_mask_and_labels():
|
| 157 |
+
base = tiny("llama").eval()
|
| 158 |
+
m = get_tensorpeft_model(base, TensorPEFTConfig(atoms_per_group=1))
|
| 159 |
+
x = batches(1)[0]
|
| 160 |
+
m.init_atoms([{"input_ids": x, "attention_mask": torch.ones_like(x), "labels": x}], verbose=False)
|
| 161 |
+
assert m.is_initialized
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def test_bf16_base():
|
| 165 |
+
base = tiny("llama").to(torch.bfloat16).eval()
|
| 166 |
+
m = get_tensorpeft_model(base, TensorPEFTConfig(atoms_per_group=2))
|
| 167 |
+
m.init_atoms(batches(2), verbose=False)
|
| 168 |
+
with torch.no_grad():
|
| 169 |
+
for g in m.groups:
|
| 170 |
+
g.alpha_raw.add_(0.01)
|
| 171 |
+
out = m(batches(1)[0]).logits
|
| 172 |
+
assert out.dtype == torch.bfloat16 and torch.isfinite(out).all()
|
| 173 |
+
assert all(p.dtype == torch.float32 for p in m.parameters() if p.requires_grad)
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def test_one_adam_lr_trains():
|
| 177 |
+
base = tiny("gpt2").eval()
|
| 178 |
+
m = get_tensorpeft_model(base, TensorPEFTConfig(atoms_per_group=2))
|
| 179 |
+
xs = batches(2, b=4, t=33)
|
| 180 |
+
m.init_atoms(xs, verbose=False)
|
| 181 |
+
opt = torch.optim.AdamW([p for p in m.parameters() if p.requires_grad], lr=3e-3, weight_decay=0.0)
|
| 182 |
+
l0 = float(lm_loss(m, xs[0]))
|
| 183 |
+
for _ in range(10):
|
| 184 |
+
opt.zero_grad()
|
| 185 |
+
lm_loss(m, xs[0]).backward()
|
| 186 |
+
opt.step()
|
| 187 |
+
assert float(lm_loss(m, xs[0])) < l0
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def test_hf_trainer_integration(tmp_path):
|
| 191 |
+
from tensorpeft import TensorPEFTTrainer
|
| 192 |
+
|
| 193 |
+
base = tiny("llama")
|
| 194 |
+
m = get_tensorpeft_model(base, TensorPEFTConfig(atoms_per_group=2))
|
| 195 |
+
data = [{"input_ids": x, "attention_mask": torch.ones_like(x), "labels": x.clone()}
|
| 196 |
+
for x in torch.randint(0, V, (16, 24)).unbind(0)]
|
| 197 |
+
m.init_atoms(torch.utils.data.DataLoader(data, batch_size=4), verbose=False)
|
| 198 |
+
|
| 199 |
+
class DS(torch.utils.data.Dataset):
|
| 200 |
+
def __len__(self):
|
| 201 |
+
return len(data)
|
| 202 |
+
|
| 203 |
+
def __getitem__(self, i):
|
| 204 |
+
return data[i]
|
| 205 |
+
|
| 206 |
+
args = T.TrainingArguments(output_dir=str(tmp_path / "out"), per_device_train_batch_size=4, max_steps=6,
|
| 207 |
+
learning_rate=3e-3, warmup_steps=1, logging_steps=1, save_steps=3,
|
| 208 |
+
report_to=[], use_cpu=True)
|
| 209 |
+
tr = TensorPEFTTrainer(model=m, args=args, train_dataset=DS())
|
| 210 |
+
hist = tr.train()
|
| 211 |
+
assert math.isfinite(hist.training_loss)
|
| 212 |
+
tr.save_model(str(tmp_path / "final"))
|
| 213 |
+
re = TensorPEFTModel.from_pretrained(tiny("llama"), str(tmp_path / "final"))
|
| 214 |
+
assert re.is_initialized
|
| 215 |
+
import os
|
| 216 |
+
assert os.path.exists(tmp_path / "out" / "checkpoint-3" / "adapter_model.safetensors")
|