Angshul commited on
Commit
49dddf5
·
verified ·
1 Parent(s): 29e60f4

Upload 9 files

Browse files
examples/reproduce_gpt2_bengali.py ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Reproduce the GPT-2 / Bengali small-budget benchmark with the package API (plain PyTorch loop).
2
+
3
+ python reproduce_gpt2_bengali.py --model <gpt2 dir or 'gpt2'> --data <dir with train.bin, val.bin> \
4
+ --atoms-per-group 1 # 1 -> 9,259 params, 4 -> 37,036, 6 -> 55,554
5
+
6
+ train.bin / val.bin are uint16 GPT-2 token ids. Reported number: mean cross-entropy (nats/token)
7
+ on the first 64 windows of 128+1 tokens of val.bin.
8
+ """
9
+ import argparse
10
+ import math
11
+ import time
12
+
13
+ import numpy as np
14
+ import torch
15
+ import torch.nn.functional as F
16
+ from transformers import AutoModelForCausalLM
17
+
18
+ from tensorpeft import TensorPEFTConfig, get_tensorpeft_model
19
+
20
+ p = argparse.ArgumentParser()
21
+ p.add_argument("--model", required=True)
22
+ p.add_argument("--data", required=True)
23
+ p.add_argument("--atoms-per-group", type=int, default=1)
24
+ p.add_argument("--steps", type=int, default=300)
25
+ p.add_argument("--lr", type=float, default=3e-3)
26
+ p.add_argument("--warmup", type=int, default=10)
27
+ p.add_argument("--batch", type=int, default=8)
28
+ p.add_argument("--seq", type=int, default=128)
29
+ p.add_argument("--init-batches", type=int, default=4)
30
+ p.add_argument("--seed", type=int, default=0)
31
+ p.add_argument("--save", default=None)
32
+ a = p.parse_args()
33
+ torch.manual_seed(a.seed)
34
+
35
+ train = np.memmap(f"{a.data}/train.bin", dtype=np.uint16, mode="r")
36
+ val = np.memmap(f"{a.data}/val.bin", dtype=np.uint16, mode="r")
37
+
38
+
39
+ def sampler(seed):
40
+ rng = np.random.default_rng(seed)
41
+ while True:
42
+ ix = rng.integers(0, len(train) - a.seq - 1, a.batch)
43
+ yield torch.from_numpy(np.stack([train[i:i + a.seq + 1] for i in ix]).astype(np.int64))
44
+
45
+
46
+ V = torch.from_numpy(np.asarray(val[:64 * (a.seq + 1)]).astype(np.int64).reshape(64, a.seq + 1))
47
+
48
+
49
+ def loss_on(model, x):
50
+ return F.cross_entropy(model(x[:, :-1]).logits.flatten(0, 1), x[:, 1:].flatten())
51
+
52
+
53
+ @torch.no_grad()
54
+ def evaluate(model):
55
+ return sum(float(loss_on(model, V[i:i + 16])) * len(V[i:i + 16]) for i in range(0, len(V), 16)) / len(V)
56
+
57
+
58
+ base = AutoModelForCausalLM.from_pretrained(a.model, dtype=torch.float32).eval()
59
+ model = get_tensorpeft_model(base, TensorPEFTConfig(atoms_per_group=a.atoms_per_group, init_batches=a.init_batches))
60
+ print("base val", round(evaluate(model), 4))
61
+ t0 = time.time()
62
+ model.init_atoms(sampler(a.seed + 12345)) # one gradient probe -> all atoms
63
+ model.print_trainable_parameters()
64
+
65
+ opt = torch.optim.AdamW([q for q in model.parameters() if q.requires_grad], lr=a.lr, weight_decay=0.0)
66
+ sched = torch.optim.lr_scheduler.LambdaLR(opt, lambda s: min(1.0, (s + 1) / a.warmup))
67
+ batches = sampler(a.seed)
68
+ for step in range(1, a.steps + 1):
69
+ loss = loss_on(model, next(batches))
70
+ opt.zero_grad(set_to_none=True)
71
+ loss.backward()
72
+ opt.step()
73
+ sched.step()
74
+ if step % 60 == 0 or step == a.steps:
75
+ print(f"step {step} train {float(loss):.4f} val {evaluate(model):.4f} {time.time() - t0:.0f}s", flush=True)
76
+ v = evaluate(model)
77
+ print(f"final val {v:.4f} (ppl {math.exp(v):.2f})")
78
+ if a.save:
79
+ model.save_pretrained(a.save)
examples/train_with_trainer.py ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Fine-tune any Hugging Face causal LM on a text file with TensorPEFT and the HF Trainer.
2
+
3
+ python train_with_trainer.py --model Qwen/Qwen2.5-0.5B --text corpus.txt --out tp_adapter \
4
+ --atoms-per-group 6 --steps 300
5
+ """
6
+ import argparse
7
+
8
+ import torch
9
+ from transformers import AutoModelForCausalLM, AutoTokenizer, DataCollatorForLanguageModeling, TrainingArguments
10
+
11
+ from tensorpeft import TensorPEFTConfig, TensorPEFTTrainer, get_tensorpeft_model
12
+
13
+ p = argparse.ArgumentParser()
14
+ p.add_argument("--model", required=True)
15
+ p.add_argument("--text", required=True, help="plain-text training file")
16
+ p.add_argument("--out", required=True)
17
+ p.add_argument("--atoms-per-group", type=int, default=4)
18
+ p.add_argument("--seq", type=int, default=256)
19
+ p.add_argument("--batch", type=int, default=8)
20
+ p.add_argument("--steps", type=int, default=300)
21
+ p.add_argument("--lr", type=float, default=3e-3)
22
+ p.add_argument("--bf16", action="store_true")
23
+ a = p.parse_args()
24
+
25
+ tok = AutoTokenizer.from_pretrained(a.model)
26
+ if tok.pad_token is None:
27
+ tok.pad_token = tok.eos_token
28
+ model = AutoModelForCausalLM.from_pretrained(a.model, dtype=torch.bfloat16 if a.bf16 else torch.float32)
29
+
30
+ ids = tok(open(a.text, encoding="utf-8").read(), return_tensors="pt").input_ids[0]
31
+ n = len(ids) // a.seq
32
+ blocks = ids[: n * a.seq].view(n, a.seq)
33
+ train = [{"input_ids": b} for b in blocks]
34
+
35
+ model = get_tensorpeft_model(model, TensorPEFTConfig(atoms_per_group=a.atoms_per_group))
36
+ collate = DataCollatorForLanguageModeling(tok, mlm=False)
37
+ loader = torch.utils.data.DataLoader(train, batch_size=a.batch, shuffle=True, collate_fn=collate)
38
+ model.init_atoms(loader, n_batches=4) # the one extra step compared with LoRA
39
+ model.print_trainable_parameters()
40
+
41
+ args = TrainingArguments(output_dir=a.out, per_device_train_batch_size=a.batch, max_steps=a.steps,
42
+ learning_rate=a.lr, warmup_steps=10, lr_scheduler_type="constant_with_warmup",
43
+ weight_decay=0.0, logging_steps=20, save_steps=100, bf16=a.bf16, report_to=[])
44
+ trainer = TensorPEFTTrainer(model=model, args=args, train_dataset=train, data_collator=collate)
45
+ trainer.train()
46
+ model.save_pretrained(a.out) # adapter only (tens of KB for small budgets)
47
+ # model.push_to_hub("your-name/your-adapter")
src/tensorpeft/__init__.py ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2026 Angshul Majumdar. Licensed under the Apache License, Version 2.0.
2
+ """TensorPEFT: parameter-efficient fine-tuning with tensor gradient-pursuit adapters."""
3
+ __version__ = "0.1.0"
4
+
5
+ from .config import TensorPEFTConfig # noqa: E402
6
+ from .model import TensorPEFTModel, get_tensorpeft_model # noqa: E402
7
+
8
+
9
+ def __getattr__(name): # lazy: importing the Trainer pulls in accelerate
10
+ if name == "TensorPEFTTrainer":
11
+ from .trainer import TensorPEFTTrainer
12
+ return TensorPEFTTrainer
13
+ raise AttributeError(name)
14
+
15
+
16
+ __all__ = ["TensorPEFTConfig", "TensorPEFTModel", "TensorPEFTTrainer", "get_tensorpeft_model", "__version__"]
src/tensorpeft/config.py ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2026 Angshul Majumdar. Licensed under the Apache License, Version 2.0.
2
+ """TensorPEFTConfig."""
3
+ import json
4
+ import os
5
+ from dataclasses import asdict, dataclass, field
6
+ from typing import List, Optional, Union
7
+
8
+ CONFIG_NAME = "adapter_config.json"
9
+ WEIGHTS_NAME = "adapter_model.safetensors"
10
+
11
+
12
+ @dataclass
13
+ class TensorPEFTConfig:
14
+ """Configuration of a TensorPEFT adapter.
15
+
16
+ Args:
17
+ atoms_per_group: rank-one CP atoms per tensor group (all chosen at initialisation).
18
+ Trainable parameters = sum over groups of atoms_per_group * (1 + sum of the group's mode sizes).
19
+ target_modules: ``"all-linear"`` (every Linear / Conv1D inside the transformer blocks) or a
20
+ list of module-name suffixes, e.g. ``["q_proj", "v_proj"]`` (``"c_attn.q"`` etc. for GPT-2).
21
+ exclude_modules: substrings of module names that are never adapted.
22
+ split_fused_qkv: split GPT-2-style fused ``c_attn`` = [q|k|v] into q, k, v.
23
+ hopm_iters: higher-order power iterations used to compute each atom.
24
+ alpha_scale: coefficients are parametrised as ``alpha_scale * alpha_raw`` so that a single
25
+ optimiser learning rate (about 3e-3 for Adam/AdamW) suits both coefficients and factors.
26
+ init_batches: default number of batches used by ``init_atoms``.
27
+ """
28
+ atoms_per_group: int = 4
29
+ target_modules: Union[str, List[str]] = "all-linear"
30
+ exclude_modules: List[str] = field(default_factory=lambda: ["lm_head", "embed", "wte", "wpe"])
31
+ split_fused_qkv: bool = True
32
+ hopm_iters: int = 30
33
+ alpha_scale: float = 100.0 / 3.0
34
+ init_batches: int = 4
35
+ base_model_name_or_path: Optional[str] = None
36
+ peft_type: str = "TENSORPEFT"
37
+ task_type: str = "CAUSAL_LM"
38
+ tensorpeft_version: Optional[str] = None
39
+
40
+ def to_dict(self):
41
+ return asdict(self)
42
+
43
+ def save_pretrained(self, save_directory, extra=None):
44
+ os.makedirs(save_directory, exist_ok=True)
45
+ d = self.to_dict()
46
+ if extra:
47
+ d.update(extra)
48
+ with open(os.path.join(save_directory, CONFIG_NAME), "w") as f:
49
+ json.dump(d, f, indent=2)
50
+
51
+ @classmethod
52
+ def from_dict(cls, d):
53
+ names = {f for f in cls.__dataclass_fields__}
54
+ return cls(**{k: v for k, v in d.items() if k in names})
55
+
56
+ @classmethod
57
+ def from_pretrained(cls, model_id, **hub_kwargs):
58
+ path = _resolve(model_id, CONFIG_NAME, **hub_kwargs)
59
+ with open(path) as f:
60
+ d = json.load(f)
61
+ return cls.from_dict(d), d
62
+
63
+
64
+ def _resolve(model_id, filename, **hub_kwargs):
65
+ local = os.path.join(model_id, filename)
66
+ if os.path.isfile(local):
67
+ return local
68
+ from huggingface_hub import hf_hub_download
69
+ return hf_hub_download(repo_id=model_id, filename=filename, **hub_kwargs)
src/tensorpeft/cp.py ADDED
@@ -0,0 +1,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2026 Angshul Majumdar. Licensed under the Apache License, Version 2.0.
2
+ """Rank-one CP approximation by higher-order power iteration (HOPM)."""
3
+ import torch
4
+
5
+ _L = "abcdefgh"
6
+
7
+
8
+ def contract_except(T, us, n):
9
+ N = T.dim()
10
+ idx = _L[:N]
11
+ spec = idx + "," + ",".join(idx[m] for m in range(N) if m != n) + "->" + idx[n]
12
+ return torch.einsum(spec, T, *[us[m] for m in range(N) if m != n])
13
+
14
+
15
+ def cp_value(T, us):
16
+ idx = _L[: T.dim()]
17
+ return torch.einsum(idx + "," + ",".join(idx) + "->", T, *us)
18
+
19
+
20
+ def outer(us):
21
+ out = us[0]
22
+ for u in us[1:]:
23
+ out = out[..., None] * u
24
+ return out
25
+
26
+
27
+ @torch.no_grad()
28
+ def rank1_cp(T, iters=30, tol=1e-6):
29
+ """Unit factors u_1..u_N (approximately) maximising sigma = <T, u_1 o ... o u_N>.
30
+
31
+ T has shape (leading..., d_in, d_out) with one or two leading modes. Initialised from the top
32
+ singular pair of the largest (d_in x d_out) slice (deterministic), then HOPM sweeps."""
33
+ N = T.dim()
34
+ lead_shape = T.shape[:-2]
35
+ slices = T.reshape(-1, T.shape[-2], T.shape[-1])
36
+ j = int(slices.flatten(1).norm(dim=1).argmax())
37
+ M = slices[j]
38
+ small = M @ M.T if M.shape[0] <= M.shape[1] else M.T @ M # exact top pair via Gram eigvec
39
+ _, vecs = torch.linalg.eigh(small.double())
40
+ v = vecs[:, -1].to(T.dtype)
41
+ if M.shape[0] <= M.shape[1]:
42
+ u_in, u_out = v, M.T @ v
43
+ else:
44
+ u_in, u_out = M @ v, v
45
+ u_in, u_out = u_in / u_in.norm().clamp_min(1e-30), u_out / u_out.norm().clamp_min(1e-30)
46
+ core = torch.einsum("sij,i,j->s", slices, u_in, u_out).reshape(lead_shape)
47
+ if core.dim() == 1:
48
+ us = [core / core.norm().clamp_min(1e-30)]
49
+ else:
50
+ Uc, _, Vc = torch.linalg.svd(core, full_matrices=False)
51
+ us = [Uc[:, 0], Vc[0]]
52
+ us = us + [u_in, u_out]
53
+ sigma = cp_value(T, us)
54
+ for _ in range(iters):
55
+ for n in range(N):
56
+ w = contract_except(T, us, n)
57
+ us[n] = w / w.norm().clamp_min(1e-30)
58
+ new = cp_value(T, us)
59
+ done = abs(float(new) - float(sigma)) <= tol * max(abs(float(new)), 1e-30)
60
+ sigma = new
61
+ if done:
62
+ break
63
+ if sigma < 0:
64
+ us[-1] = -us[-1]
65
+ sigma = -sigma
66
+ return us, float(sigma)
src/tensorpeft/layers.py ADDED
@@ -0,0 +1,103 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2026 Angshul Majumdar. Licensed under the Apache License, Version 2.0.
2
+ """Discovery of adaptable weight matrices ('slots') and the forward hooks that apply dW."""
3
+ import re
4
+ from collections import defaultdict
5
+
6
+ import torch
7
+ import torch.nn as nn
8
+
9
+ try:
10
+ from transformers.pytorch_utils import Conv1D
11
+ except Exception: # pragma: no cover
12
+ class Conv1D(nn.Module):
13
+ pass
14
+
15
+ # GPT-2 family: Conv1D c_attn holds [q | k | v] as three equal column blocks
16
+ FUSED_EQUAL_SPLIT = {"c_attn": ("q", "k", "v")}
17
+ _LAYER_RE = re.compile(r"\.(\d+)\.")
18
+
19
+
20
+ class Slot:
21
+ """One adapted matrix, always viewed as (d_in, d_out) so that the adapter adds x @ dW."""
22
+
23
+ def __init__(self, module_name, module, layer, role, c0, c1, d_in):
24
+ self.module_name, self.module = module_name, module
25
+ self.layer, self.role = layer, role
26
+ self.c0, self.c1, self.d_in = c0, c1, d_in
27
+ self.conv1d = isinstance(module, Conv1D)
28
+
29
+ @property
30
+ def shape(self):
31
+ return (self.d_in, self.c1 - self.c0)
32
+
33
+ def _view(self, w):
34
+ return w[:, self.c0:self.c1] if self.conv1d else w[self.c0:self.c1, :].T
35
+
36
+ def weight(self):
37
+ return self._view(self.module.weight)
38
+
39
+ def grad(self):
40
+ return self._view(self.module.weight.grad)
41
+
42
+ def __repr__(self):
43
+ return f"Slot(layer {self.layer}: {self.role} {list(self.shape)})"
44
+
45
+
46
+ def find_slots(model, target_modules="all-linear", exclude_modules=(), split_fused_qkv=True):
47
+ slots = []
48
+ for name, mod in model.named_modules():
49
+ if not isinstance(mod, (nn.Linear, Conv1D)):
50
+ continue
51
+ if any(e in name for e in exclude_modules):
52
+ continue
53
+ m = _LAYER_RE.search(name)
54
+ if m is None:
55
+ continue
56
+ role = name[m.end():]
57
+ w = mod.weight
58
+ d_in, d_out = (w.shape[0], w.shape[1]) if isinstance(mod, Conv1D) else (w.shape[1], w.shape[0])
59
+ parts = FUSED_EQUAL_SPLIT.get(role.split(".")[-1]) if (split_fused_qkv and isinstance(mod, Conv1D)) else None
60
+ if parts and d_out % len(parts) == 0:
61
+ wd = d_out // len(parts)
62
+ for j, p in enumerate(parts):
63
+ slots.append(Slot(name, mod, int(m.group(1)), f"{role}.{p}", j * wd, (j + 1) * wd, d_in))
64
+ else:
65
+ slots.append(Slot(name, mod, int(m.group(1)), role, 0, d_out, d_in))
66
+ if target_modules != "all-linear":
67
+ if isinstance(target_modules, str):
68
+ target_modules = [target_modules]
69
+ pats = [re.compile(r"(^|\.)" + re.escape(t) + r"$") for t in target_modules]
70
+ slots = [s for s in slots if any(p.search(s.role) for p in pats)]
71
+ if not slots:
72
+ raise ValueError(f"TensorPEFT: no target modules matched {target_modules!r}")
73
+ return slots
74
+
75
+
76
+ def attach_hooks(slots, delta_fn):
77
+ """Forward hooks adding delta_fn(slot, x) to each slot's output columns (cast to the output dtype)."""
78
+ by_mod = defaultdict(list)
79
+ for s in slots:
80
+ by_mod[id(s.module)].append(s)
81
+ handles = []
82
+ for ss in by_mod.values():
83
+ ss = sorted(ss, key=lambda s: s.c0)
84
+ mod = ss[0].module
85
+ width = mod.weight.shape[1] if ss[0].conv1d else mod.weight.shape[0]
86
+
87
+ def hook(module, inputs, out, ss=ss, width=width):
88
+ x = inputs[0]
89
+ deltas = [delta_fn(s, x) for s in ss]
90
+ if all(d is None for d in deltas):
91
+ return out
92
+ pieces, pos = [], 0
93
+ for s, d in zip(ss, deltas):
94
+ if s.c0 > pos:
95
+ pieces.append(out.new_zeros(*out.shape[:-1], s.c0 - pos))
96
+ pieces.append(out.new_zeros(*out.shape[:-1], s.c1 - s.c0) if d is None else d.to(out.dtype))
97
+ pos = s.c1
98
+ if pos < width:
99
+ pieces.append(out.new_zeros(*out.shape[:-1], width - pos))
100
+ return out + (pieces[0] if len(pieces) == 1 else torch.cat(pieces, dim=-1))
101
+
102
+ handles.append(mod.register_forward_hook(hook))
103
+ return handles
src/tensorpeft/model.py ADDED
@@ -0,0 +1,307 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2026 Angshul Majumdar. Licensed under the Apache License, Version 2.0.
2
+ """TensorPEFT: tensor gradient-pursuit adapters.
3
+
4
+ Matrices of equal shape across all transformer blocks are stacked into one update tensor
5
+ (layer x [role] x d_in x d_out). Each group's update is a sum of rank-one CP atoms
6
+
7
+ dW = sum_i a_i u_i(layer) o u_i(role) o u_i(in) o u_i(out).
8
+
9
+ ``init_atoms`` runs one gradient probe, and the atoms are the leading rank-one CP components of
10
+ the negative gradient tensor (HOPM with deflation). Coefficients start at zero, so the model is
11
+ unchanged until training starts; all coefficients and factors are then trained normally.
12
+ """
13
+ import functools
14
+ import os
15
+
16
+ import torch
17
+ import torch.nn as nn
18
+
19
+ from . import __version__
20
+ from .config import CONFIG_NAME, WEIGHTS_NAME, TensorPEFTConfig, _resolve
21
+ from .cp import cp_value, outer, rank1_cp
22
+ from .layers import attach_hooks, find_slots
23
+
24
+
25
+ class TensorGroup(nn.Module):
26
+ def __init__(self, slots, k, alpha_scale):
27
+ super().__init__()
28
+ self.slots = slots
29
+ self.d_in, self.d_out = slots[0].shape
30
+ self.layers = sorted({s.layer for s in slots})
31
+ self.roles = sorted({s.role for s in slots})
32
+ self.has_p = len(self.roles) > 1
33
+ self.shape = (len(self.layers),) + ((len(self.roles),) if self.has_p else ()) + (self.d_in, self.d_out)
34
+ self.index = {id(s): (self.layers.index(s.layer),) + ((self.roles.index(s.role),) if self.has_p else ())
35
+ for s in slots}
36
+ self.k = k
37
+ self.alpha_scale = alpha_scale
38
+ dev = slots[0].module.weight.device
39
+ self.alpha_raw = nn.Parameter(torch.zeros(k, device=dev))
40
+ self.factors = nn.ParameterList([nn.Parameter(torch.zeros(k, n, device=dev)) for n in self.shape])
41
+ self.register_buffer("initialized", torch.tensor(False, device=dev))
42
+ self._cache = None
43
+
44
+ @property
45
+ def name(self):
46
+ return f"{'+'.join(self.roles)} {list(self.shape)}"
47
+
48
+ def n_params(self):
49
+ return self.k * (1 + sum(self.shape))
50
+
51
+ def meta(self):
52
+ return {"roles": self.roles, "layers": self.layers, "shape": list(self.shape)}
53
+
54
+ def prepare(self):
55
+ if not bool(self.initialized):
56
+ self._cache = None
57
+ return
58
+ a = self.alpha_raw * self.alpha_scale
59
+ F = list(self.factors)
60
+ C = torch.einsum("k,kl,kp->klp", a, F[0], F[1]) if self.has_p else a[:, None] * F[0]
61
+ self._cache = (C, F[-2], F[-1])
62
+
63
+ def delta(self, slot, x):
64
+ if self._cache is None:
65
+ return None
66
+ C, U_in, U_out = self._cache
67
+ c = C[(slice(None),) + self.index[id(slot)]]
68
+ return ((x.to(U_in.dtype) @ U_in.T) * c) @ U_out
69
+
70
+ @torch.no_grad()
71
+ def dense_delta(self, slot):
72
+ self.prepare()
73
+ if self._cache is None:
74
+ return None
75
+ C, U_in, U_out = self._cache
76
+ c = C[(slice(None),) + self.index[id(slot)]]
77
+ return (U_in.T * c) @ U_out
78
+
79
+ @torch.no_grad()
80
+ def stacked_grad(self):
81
+ w = self.slots[0].module.weight
82
+ G = torch.zeros(self.shape, dtype=torch.float32, device=w.device)
83
+ for s in self.slots:
84
+ G[self.index[id(s)]] = s.grad().float()
85
+ return G
86
+
87
+
88
+ class TensorPEFTModel(nn.Module):
89
+ """Wraps a Hugging Face causal LM. Use :func:`get_tensorpeft_model` to create one."""
90
+
91
+ def __init__(self, base_model, config: TensorPEFTConfig):
92
+ super().__init__()
93
+ self.base_model = base_model
94
+ self.peft_config = config
95
+ config.tensorpeft_version = __version__
96
+ if config.base_model_name_or_path is None:
97
+ config.base_model_name_or_path = getattr(getattr(base_model, "config", None), "_name_or_path", None)
98
+ for p in base_model.parameters():
99
+ p.requires_grad_(False)
100
+ self.slots = find_slots(base_model, config.target_modules, config.exclude_modules, config.split_fused_qkv)
101
+ by_shape = {}
102
+ for s in self.slots:
103
+ by_shape.setdefault(s.shape, []).append(s)
104
+ self.groups = nn.ModuleList([TensorGroup(ss, config.atoms_per_group, config.alpha_scale)
105
+ for ss in by_shape.values()])
106
+ self._group_of = {id(s): g for g in self.groups for s in g.slots}
107
+ self._handles = attach_hooks(self.slots, lambda s, x: self._group_of[id(s)].delta(s, x))
108
+ self._handles.append(base_model.register_forward_pre_hook(lambda m, a: self._prepare()))
109
+ self._modules_w = list({id(s.module): s.module for s in self.slots}.values())
110
+
111
+ # forward with the base model's signature (so HF Trainer keeps the right dataset columns)
112
+ def forward(*args, **kwargs):
113
+ return self.base_model(*args, **kwargs)
114
+ functools.update_wrapper(forward, base_model.forward)
115
+ self.forward = forward
116
+
117
+ # ------------------------------------------------------------------ plumbing
118
+ def _prepare(self):
119
+ for g in self.groups:
120
+ g.prepare()
121
+
122
+ def __getattr__(self, name):
123
+ try:
124
+ return super().__getattr__(name)
125
+ except AttributeError:
126
+ return getattr(super().__getattr__("base_model"), name)
127
+
128
+ def get_base_model(self):
129
+ return self.base_model
130
+
131
+ @property
132
+ def is_initialized(self):
133
+ return all(bool(g.initialized) for g in self.groups)
134
+
135
+ def get_nb_trainable_parameters(self):
136
+ trainable = sum(p.numel() for p in self.parameters() if p.requires_grad)
137
+ total = sum(p.numel() for p in self.base_model.parameters()) + trainable
138
+ return trainable, total
139
+
140
+ def print_trainable_parameters(self):
141
+ t, a = self.get_nb_trainable_parameters()
142
+ print(f"trainable params: {t:,} || all params: {a:,} || trainable%: {100 * t / a:.4f}")
143
+
144
+ def describe(self):
145
+ return [f"{g.name}: {g.k} atoms, {g.n_params():,} params" for g in self.groups]
146
+
147
+ # ------------------------------------------------------------------ initialisation
148
+ def init_atoms(self, batches, n_batches=None, loss_fn=None, verbose=True):
149
+ """Choose all atoms from one gradient probe.
150
+
151
+ batches: iterable of dicts (``input_ids``, optional ``attention_mask`` / ``labels``) or of
152
+ input-id tensors (causal-LM labels = inputs). A DataLoader works.
153
+ n_batches: how many batches to use (default ``config.init_batches``).
154
+ loss_fn: optional ``loss_fn(base_model, batch) -> scalar``.
155
+ Returns, per group, each atom's initial descent rate <-G, atom> (= -dL/da at a = 0).
156
+ """
157
+ n_batches = n_batches or self.peft_config.init_batches
158
+ dev = self._modules_w[0].weight.device
159
+ was_training = self.base_model.training
160
+ self.base_model.eval()
161
+ for m in self._modules_w:
162
+ m.weight.requires_grad_(True)
163
+ m.weight.grad = None
164
+ used = 0
165
+ for b in batches:
166
+ if used >= n_batches:
167
+ break
168
+ if loss_fn is not None:
169
+ loss = loss_fn(self.base_model, b)
170
+ else:
171
+ if torch.is_tensor(b):
172
+ b = {"input_ids": b}
173
+ b = {k: (v.to(dev) if torch.is_tensor(v) else v) for k, v in b.items()}
174
+ if "labels" not in b:
175
+ b["labels"] = b["input_ids"]
176
+ loss = self.base_model(**b).loss
177
+ loss.backward()
178
+ used += 1
179
+ if used == 0:
180
+ raise ValueError("init_atoms: no batches given")
181
+ report = []
182
+ with torch.no_grad():
183
+ for g in self.groups:
184
+ negG = -g.stacked_grad() / used # mean gradient over the batches used
185
+ T = negG.clone()
186
+ sig = []
187
+ for i in range(g.k):
188
+ us, s = rank1_cp(T, iters=self.peft_config.hopm_iters)
189
+ for pl, u in zip(g.factors, us):
190
+ pl[i].copy_(u)
191
+ T = T - s * outer(us) # deflation
192
+ sig.append(float(cp_value(negG, us)))
193
+ del negG, T
194
+ g.alpha_raw.zero_()
195
+ g.initialized.fill_(True)
196
+ report.append(sig)
197
+ for m in self._modules_w:
198
+ m.weight.grad = None
199
+ m.weight.requires_grad_(False)
200
+ for p in self.parameters():
201
+ p.grad = None
202
+ self.base_model.train(was_training)
203
+ if verbose:
204
+ for g, s in zip(self.groups, report):
205
+ print(f"TensorPEFT {g.name}: {g.k} atoms, initial descent rate {s[0]:.4g} .. {s[-1]:.4g}")
206
+ return report
207
+
208
+ # ------------------------------------------------------------------ save / load
209
+ def adapter_state_dict(self):
210
+ sd = {}
211
+ for gi, g in enumerate(self.groups):
212
+ sd[f"group{gi}.alpha_raw"] = g.alpha_raw.detach().cpu().contiguous()
213
+ for m, pl in enumerate(g.factors):
214
+ sd[f"group{gi}.factor{m}"] = pl.detach().cpu().contiguous()
215
+ return sd
216
+
217
+ def save_pretrained(self, save_directory, **kwargs):
218
+ if not self.is_initialized:
219
+ raise RuntimeError("call init_atoms(...) before saving")
220
+ from safetensors.torch import save_file
221
+ os.makedirs(save_directory, exist_ok=True)
222
+ self.peft_config.save_pretrained(save_directory, extra={"groups": [g.meta() for g in self.groups]})
223
+ save_file(self.adapter_state_dict(), os.path.join(save_directory, WEIGHTS_NAME),
224
+ metadata={"format": "pt", "library": "tensorpeft"})
225
+ readme = os.path.join(save_directory, "README.md")
226
+ if not os.path.exists(readme):
227
+ with open(readme, "w") as f:
228
+ f.write(_model_card(self))
229
+
230
+ @classmethod
231
+ def from_pretrained(cls, base_model, model_id, is_trainable=False, **hub_kwargs):
232
+ config, raw = TensorPEFTConfig.from_pretrained(model_id, **hub_kwargs)
233
+ model = cls(base_model, config)
234
+ saved = raw.get("groups")
235
+ if saved is not None:
236
+ mine = [g.meta() for g in model.groups]
237
+ if [(m["roles"], m["shape"]) for m in mine] != [(m["roles"], m["shape"]) for m in saved]:
238
+ raise ValueError("adapter does not match this base model's architecture")
239
+ from safetensors.torch import load_file
240
+ sd = load_file(_resolve(model_id, WEIGHTS_NAME, **hub_kwargs))
241
+ with torch.no_grad():
242
+ for gi, g in enumerate(model.groups):
243
+ g.alpha_raw.copy_(sd[f"group{gi}.alpha_raw"])
244
+ for m, pl in enumerate(g.factors):
245
+ pl.copy_(sd[f"group{gi}.factor{m}"])
246
+ g.initialized.fill_(True)
247
+ for p in model.parameters():
248
+ if p.requires_grad and not is_trainable:
249
+ p.requires_grad_(False)
250
+ return model
251
+
252
+ def push_to_hub(self, repo_id, private=None, token=None, commit_message="Upload TensorPEFT adapter"):
253
+ import tempfile
254
+ from huggingface_hub import HfApi
255
+ api = HfApi(token=token)
256
+ api.create_repo(repo_id, private=private, exist_ok=True)
257
+ with tempfile.TemporaryDirectory() as d:
258
+ self.save_pretrained(d)
259
+ return api.upload_folder(repo_id=repo_id, folder_path=d, commit_message=commit_message)
260
+
261
+ # ------------------------------------------------------------------ merge
262
+ @torch.no_grad()
263
+ def merge_and_unload(self):
264
+ """Write dW into the base weights and return the plain base model (no inference overhead)."""
265
+ for s in self.slots:
266
+ d = self._group_of[id(s)].dense_delta(s)
267
+ if d is not None:
268
+ w = s.weight()
269
+ w.add_(d.to(w.dtype))
270
+ return self.unload()
271
+
272
+ def unload(self):
273
+ """Remove the adapter hooks without merging and return the base model."""
274
+ for h in self._handles:
275
+ h.remove()
276
+ self._handles = []
277
+ return self.base_model
278
+
279
+
280
+ def get_tensorpeft_model(model, config: TensorPEFTConfig = None):
281
+ """Wrap a Hugging Face model with TensorPEFT. Call ``.init_atoms(batches)`` before training."""
282
+ return TensorPEFTModel(model, config or TensorPEFTConfig())
283
+
284
+
285
+ def _model_card(m):
286
+ c = m.peft_config
287
+ t, _ = m.get_nb_trainable_parameters()
288
+ return f"""---
289
+ library_name: tensorpeft
290
+ base_model: {c.base_model_name_or_path or 'unknown'}
291
+ tags:
292
+ - tensorpeft
293
+ - peft
294
+ ---
295
+
296
+ # TensorPEFT adapter
297
+
298
+ Base model: `{c.base_model_name_or_path}`; {c.atoms_per_group} atoms per tensor group; {t:,} adapter parameters.
299
+
300
+ ```python
301
+ from transformers import AutoModelForCausalLM
302
+ from tensorpeft import TensorPEFTModel
303
+ base = AutoModelForCausalLM.from_pretrained("{c.base_model_name_or_path}")
304
+ model = TensorPEFTModel.from_pretrained(base, "<this repo id>")
305
+ model = model.merge_and_unload() # optional
306
+ ```
307
+ """
src/tensorpeft/trainer.py ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2026 Angshul Majumdar. Licensed under the Apache License, Version 2.0.
2
+ """Hugging Face Trainer that saves TensorPEFT checkpoints as adapters (not the full model)."""
3
+ import os
4
+
5
+ from transformers import Trainer
6
+
7
+
8
+ class TensorPEFTTrainer(Trainer):
9
+ """Drop-in ``transformers.Trainer``; checkpoints and ``save_model`` write only the adapter."""
10
+
11
+ def _save(self, output_dir=None, state_dict=None):
12
+ output_dir = output_dir if output_dir is not None else self.args.output_dir
13
+ os.makedirs(output_dir, exist_ok=True)
14
+ model = self.accelerator.unwrap_model(self.model)
15
+ if self.args.should_save:
16
+ model.save_pretrained(output_dir)
17
+ if self.processing_class is not None:
18
+ self.processing_class.save_pretrained(output_dir)
tests/test_tensorpeft.py ADDED
@@ -0,0 +1,216 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """pytest tests for TensorPEFT (CPU, tiny random models)."""
2
+ import math
3
+
4
+ import pytest
5
+ import torch
6
+ import torch.nn.functional as F
7
+ import transformers as T
8
+
9
+ from tensorpeft import TensorPEFTConfig, TensorPEFTModel, get_tensorpeft_model
10
+ from tensorpeft.cp import cp_value, contract_except, outer, rank1_cp
11
+
12
+ V = 97
13
+
14
+
15
+ def tiny(arch):
16
+ common = dict(vocab_size=V, max_position_embeddings=64)
17
+ if arch == "gpt2":
18
+ return T.GPT2LMHeadModel(T.GPT2Config(n_layer=3, n_embd=32, n_head=4, vocab_size=V, n_positions=64))
19
+ if arch == "llama":
20
+ return T.LlamaForCausalLM(T.LlamaConfig(num_hidden_layers=3, hidden_size=32, intermediate_size=48,
21
+ num_attention_heads=4, num_key_value_heads=2, **common))
22
+ if arch == "qwen2":
23
+ return T.Qwen2ForCausalLM(T.Qwen2Config(num_hidden_layers=3, hidden_size=32, intermediate_size=48,
24
+ num_attention_heads=4, num_key_value_heads=2, **common))
25
+ if arch == "mistral":
26
+ return T.MistralForCausalLM(T.MistralConfig(num_hidden_layers=3, hidden_size=32, intermediate_size=48,
27
+ num_attention_heads=4, num_key_value_heads=2, **common))
28
+ if arch == "phi":
29
+ return T.PhiForCausalLM(T.PhiConfig(num_hidden_layers=3, hidden_size=32, intermediate_size=48,
30
+ num_attention_heads=4, **common))
31
+ if arch == "gemma":
32
+ return T.GemmaForCausalLM(T.GemmaConfig(num_hidden_layers=3, hidden_size=32, intermediate_size=48,
33
+ num_attention_heads=4, num_key_value_heads=2, head_dim=8, **common))
34
+ raise ValueError(arch)
35
+
36
+
37
+ ARCHS = ["gpt2", "llama", "qwen2", "mistral", "phi", "gemma"]
38
+
39
+
40
+ def batches(n=2, b=3, t=17, seed=0):
41
+ g = torch.Generator().manual_seed(seed)
42
+ return [torch.randint(0, V, (b, t), generator=g) for _ in range(n)]
43
+
44
+
45
+ def lm_loss(model, x):
46
+ logits = model(x[:, :-1]).logits
47
+ return F.cross_entropy(logits.float().flatten(0, 1), x[:, 1:].flatten())
48
+
49
+
50
+ @pytest.fixture(autouse=True)
51
+ def _seed():
52
+ torch.manual_seed(0)
53
+
54
+
55
+ # ---------------------------------------------------------------- CP / HOPM
56
+ @pytest.mark.parametrize("shape", [(5, 30, 40), (6, 4, 30, 20)])
57
+ def test_hopm_recovers_planted_rank1(shape):
58
+ us = [F.normalize(torch.randn(n, dtype=torch.float64), dim=0) for n in shape]
59
+ T_ = 5.0 * outer(us) + 0.05 * torch.randn(shape, dtype=torch.float64)
60
+ est, sig = rank1_cp(T_, iters=50)
61
+ assert min(abs(float(a @ b)) for a, b in zip(us, est)) > 0.99
62
+ assert abs(sig - 5.0) < 0.3
63
+
64
+
65
+ def test_hopm_fixed_point():
66
+ T_ = torch.randn(4, 3, 20, 25, dtype=torch.float64)
67
+ est, sig = rank1_cp(T_, iters=500, tol=1e-14)
68
+ res = max(float((contract_except(T_, est, n) / sig - est[n]).abs().max()) for n in range(4))
69
+ assert res < 1e-6
70
+
71
+
72
+ # ---------------------------------------------------------------- model wrapping
73
+ @pytest.mark.parametrize("arch", ARCHS)
74
+ def test_unchanged_before_and_after_init(arch):
75
+ base = tiny(arch).eval()
76
+ x = batches(1)[0]
77
+ ref = base(x).logits.detach().clone()
78
+ model = get_tensorpeft_model(base, TensorPEFTConfig(atoms_per_group=2))
79
+ assert torch.equal(model(x).logits, ref)
80
+ model.init_atoms(batches(2), verbose=False)
81
+ assert model.is_initialized
82
+ assert float((model(x).logits - ref).abs().max()) < 1e-6 # a = 0 at init
83
+
84
+
85
+ @pytest.mark.parametrize("arch", ARCHS)
86
+ def test_initial_gradient_is_steepest_cp_direction(arch):
87
+ """dL/d alpha_raw_i at init = -alpha_scale * sigma_i (atoms are descent directions)."""
88
+ torch.set_default_dtype(torch.float64)
89
+ try:
90
+ base = tiny(arch).eval()
91
+ model = get_tensorpeft_model(base, TensorPEFTConfig(atoms_per_group=3))
92
+ xs = batches(2)
93
+ sig = model.init_atoms(xs, verbose=False)
94
+ loss = sum(lm_loss(model, x) for x in xs) / len(xs)
95
+ grads = torch.autograd.grad(loss, [g.alpha_raw for g in model.groups])
96
+ for g, gr, s in zip(model.groups, grads, sig):
97
+ expect = -g.alpha_scale * torch.tensor(s, dtype=gr.dtype)
98
+ assert torch.allclose(gr, expect, rtol=1e-6, atol=1e-10), (arch, gr, expect)
99
+ finally:
100
+ torch.set_default_dtype(torch.float32)
101
+
102
+
103
+ @pytest.mark.parametrize("arch", ARCHS)
104
+ def test_merge_equals_hooks_and_save_load_roundtrip(arch, tmp_path):
105
+ torch.set_default_dtype(torch.float64)
106
+ try:
107
+ base = tiny(arch).eval()
108
+ state = {k: v.clone() for k, v in base.state_dict().items()}
109
+ x = batches(1, seed=5)[0]
110
+ ref = base(x).logits.detach().clone()
111
+ model = get_tensorpeft_model(base, TensorPEFTConfig(atoms_per_group=2))
112
+ model.init_atoms(batches(2), verbose=False)
113
+ with torch.no_grad():
114
+ for p in model.parameters():
115
+ if p.requires_grad:
116
+ p.add_(0.3 * torch.randn_like(p))
117
+ hooked = model(x).logits.detach().clone()
118
+ assert float((hooked - ref).abs().max()) > 1e-3 # the adapter does change the model
119
+ model.save_pretrained(tmp_path)
120
+ # reload onto a fresh copy of the same base
121
+ base2 = tiny(arch).eval()
122
+ base2.load_state_dict(state)
123
+ re = TensorPEFTModel.from_pretrained(base2, str(tmp_path))
124
+ assert float((re(x).logits - hooked).abs().max()) < 1e-10
125
+ merged = model.merge_and_unload()
126
+ assert float((merged(x).logits - hooked).abs().max()) < 1e-10
127
+ finally:
128
+ torch.set_default_dtype(torch.float32)
129
+
130
+
131
+ def test_param_count_and_grouping_gpt2():
132
+ base = tiny("gpt2")
133
+ m = get_tensorpeft_model(base, TensorPEFTConfig(atoms_per_group=4))
134
+ names = [g.name for g in m.groups]
135
+ assert names[0].startswith("attn.c_attn.k+attn.c_attn.q+attn.c_attn.v+attn.c_proj [3, 4, 32, 32]")
136
+ expect = 4 * ((1 + 3 + 4 + 32 + 32) + (1 + 3 + 32 + 128) + (1 + 3 + 128 + 32))
137
+ assert m.get_nb_trainable_parameters()[0] == expect
138
+ assert all(not p.requires_grad for p in base.parameters())
139
+
140
+
141
+ def test_target_modules_subset():
142
+ base = tiny("llama")
143
+ m = get_tensorpeft_model(base, TensorPEFTConfig(target_modules=["q_proj", "v_proj"], atoms_per_group=1))
144
+ roles = sorted({s.role for s in m.slots})
145
+ assert roles == ["self_attn.q_proj", "self_attn.v_proj"]
146
+
147
+
148
+ def test_wrong_architecture_rejected(tmp_path):
149
+ m = get_tensorpeft_model(tiny("llama"), TensorPEFTConfig(atoms_per_group=1))
150
+ m.init_atoms(batches(1), verbose=False)
151
+ m.save_pretrained(tmp_path)
152
+ with pytest.raises(ValueError):
153
+ TensorPEFTModel.from_pretrained(tiny("gpt2"), str(tmp_path))
154
+
155
+
156
+ def test_dict_batches_with_attention_mask_and_labels():
157
+ base = tiny("llama").eval()
158
+ m = get_tensorpeft_model(base, TensorPEFTConfig(atoms_per_group=1))
159
+ x = batches(1)[0]
160
+ m.init_atoms([{"input_ids": x, "attention_mask": torch.ones_like(x), "labels": x}], verbose=False)
161
+ assert m.is_initialized
162
+
163
+
164
+ def test_bf16_base():
165
+ base = tiny("llama").to(torch.bfloat16).eval()
166
+ m = get_tensorpeft_model(base, TensorPEFTConfig(atoms_per_group=2))
167
+ m.init_atoms(batches(2), verbose=False)
168
+ with torch.no_grad():
169
+ for g in m.groups:
170
+ g.alpha_raw.add_(0.01)
171
+ out = m(batches(1)[0]).logits
172
+ assert out.dtype == torch.bfloat16 and torch.isfinite(out).all()
173
+ assert all(p.dtype == torch.float32 for p in m.parameters() if p.requires_grad)
174
+
175
+
176
+ def test_one_adam_lr_trains():
177
+ base = tiny("gpt2").eval()
178
+ m = get_tensorpeft_model(base, TensorPEFTConfig(atoms_per_group=2))
179
+ xs = batches(2, b=4, t=33)
180
+ m.init_atoms(xs, verbose=False)
181
+ opt = torch.optim.AdamW([p for p in m.parameters() if p.requires_grad], lr=3e-3, weight_decay=0.0)
182
+ l0 = float(lm_loss(m, xs[0]))
183
+ for _ in range(10):
184
+ opt.zero_grad()
185
+ lm_loss(m, xs[0]).backward()
186
+ opt.step()
187
+ assert float(lm_loss(m, xs[0])) < l0
188
+
189
+
190
+ def test_hf_trainer_integration(tmp_path):
191
+ from tensorpeft import TensorPEFTTrainer
192
+
193
+ base = tiny("llama")
194
+ m = get_tensorpeft_model(base, TensorPEFTConfig(atoms_per_group=2))
195
+ data = [{"input_ids": x, "attention_mask": torch.ones_like(x), "labels": x.clone()}
196
+ for x in torch.randint(0, V, (16, 24)).unbind(0)]
197
+ m.init_atoms(torch.utils.data.DataLoader(data, batch_size=4), verbose=False)
198
+
199
+ class DS(torch.utils.data.Dataset):
200
+ def __len__(self):
201
+ return len(data)
202
+
203
+ def __getitem__(self, i):
204
+ return data[i]
205
+
206
+ args = T.TrainingArguments(output_dir=str(tmp_path / "out"), per_device_train_batch_size=4, max_steps=6,
207
+ learning_rate=3e-3, warmup_steps=1, logging_steps=1, save_steps=3,
208
+ report_to=[], use_cpu=True)
209
+ tr = TensorPEFTTrainer(model=m, args=args, train_dataset=DS())
210
+ hist = tr.train()
211
+ assert math.isfinite(hist.training_loss)
212
+ tr.save_model(str(tmp_path / "final"))
213
+ re = TensorPEFTModel.from_pretrained(tiny("llama"), str(tmp_path / "final"))
214
+ assert re.is_initialized
215
+ import os
216
+ assert os.path.exists(tmp_path / "out" / "checkpoint-3" / "adapter_model.safetensors")