File size: 7,475 Bytes
a007f4a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 | """
Commit-intent classifier: is a commit corrective, adaptive or perfective?
CodeBERT fine-tuned on Levin & Yehudai's 1,151 labelled commits. It reads the commit message and
the per-file diff, encodes each piece separately, and attention-pools the pieces into one commit
vector. This module rebuilds exactly the architecture that was trained, so the published weights
load into it unchanged.
from commit_intent import CommitIntent
clf = CommitIntent.from_pretrained("Binamra00/codebert-commit-intent")
clf.predict("Fix NPE when the config file is empty",
{"src/main/java/Config.java": patch_text})
Training code and evaluation: github.com/Binamra00/commit-intent-and-refactoring-levels
"""
import json
import re
from pathlib import Path
import torch
import torch.nn as nn
from transformers import AutoConfig, AutoModel, AutoTokenizer
DEFAULT_REPO = "Binamra00/codebert-commit-intent"
HUNK = re.compile(r"^@@ .*? @@")
class CommitClassifier(nn.Module):
"""The trained network. Parameter names match the published state dicts."""
def __init__(self, base_model, n_labels=3, dropout=0.2):
super().__init__()
# Built from the config alone: the fine-tuned encoder weights come from the state dict.
self.enc = AutoModel.from_config(AutoConfig.from_pretrained(base_model))
hid = self.enc.config.hidden_size
self.score = nn.Linear(hid, 1) # attention score per piece
self.drop = nn.Dropout(dropout)
self.out = nn.Linear(hid, n_labels)
def forward_pieces(self, window_cls, owner):
"""window_cls: (windows, hid) [CLS] vectors; owner: piece index of each window."""
n = int(owner.max()) + 1
pv = torch.zeros(n, window_cls.size(1), device=window_cls.device, dtype=window_cls.dtype)
pv.index_add_(0, owner, window_cls)
cnt = torch.bincount(owner, minlength=n).to(window_cls.dtype).unsqueeze(1)
pv = pv / cnt # windows mean-pooled within a piece
a = self.score(pv).softmax(0) # pieces attention-pooled into a commit
return self.out(self.drop((pv * a).sum(0)))
def pieces_from_patches(message, files):
"""
Render a commit the way the model was trained on it.
message : the full commit message, as git records it
files : {path: unified-diff patch} for each changed file -- GitHub's API `patch` field or one
file's section of `git diff`. Header and hunk lines are dropped; '+' lines become
ADDED, '-' lines REMOVED. Context lines are ignored.
Returns the message piece followed by one piece per file, largest first.
"""
pieces = [f"<COMMIT_MESSAGE>{(message or '').strip()}</COMMIT_MESSAGE>"]
rendered = []
for path, patch in dict(files or {}).items():
added, removed = [], []
for ln in (patch or "").split("\n"):
if HUNK.match(ln) or ln.startswith(("--- ", "+++ ")):
continue
if ln.startswith("+"):
added.append(ln[1:])
elif ln.startswith("-"):
removed.append(ln[1:])
part = [f"<FILE>{path}</FILE>"]
if removed:
part.append("<REMOVED>" + "\n".join(removed) + "</REMOVED>")
if added:
part.append("<ADDED>" + "\n".join(added) + "</ADDED>")
rendered.append("\n".join(part))
rendered.sort(key=len, reverse=True)
return pieces + rendered
class CommitIntent:
"""Calibrated predictions from a commit's message and diff."""
def __init__(self, model, tokenizer, config, device):
self.model, self.tok, self.cfg, self.device = model, tokenizer, config, device
self.labels = config["labels"]
@classmethod
def from_pretrained(cls, repo_or_dir=DEFAULT_REPO, seed=None, device=None):
"""Load from a Hugging Face repo id or a local folder holding config.json and the weights."""
from safetensors.torch import load_file
local = Path(repo_or_dir)
if local.is_dir():
fetch = lambda name: str(local / name)
else:
from huggingface_hub import hf_hub_download
fetch = lambda name: hf_hub_download(repo_or_dir, name)
config = json.load(open(fetch("config.json"), encoding="utf-8"))
seed = str(seed or config["default_seed"])
device = device or ("cuda" if torch.cuda.is_available() else "cpu")
model = CommitClassifier(config["base_model"], len(config["labels"]), config["dropout"])
state = load_file(fetch(config["weights"][seed]))
missing, unexpected = model.load_state_dict(state, strict=False)
# Older transformers versions also save a position_ids buffer; nothing else may differ.
stray = [k for k in missing + unexpected if not k.endswith("position_ids")]
if stray:
raise RuntimeError(f"weights do not match the architecture: {stray[:5]}")
model.to(device).eval()
return cls(model, AutoTokenizer.from_pretrained(config["base_model"]), config, device)
def _windows(self, text):
ids = self.tok.encode(text, add_special_tokens=False, truncation=False)
body = self.cfg["max_length"] - 2
return [ids[i:i + body] for i in range(0, max(len(ids), 1), body)] or [[]]
@torch.no_grad()
def logits(self, pieces, batch=64):
"""Raw logits for one commit given its rendered pieces."""
pieces = pieces[: self.cfg["max_pieces"]]
wins, owner = [], []
for pi, p in enumerate(pieces):
for w in self._windows(p):
wins.append(w)
owner.append(pi)
cls_vecs = []
for s in range(0, len(wins), batch):
chunk = wins[s:s + batch]
L = max(len(w) for w in chunk) + 2
ids = torch.full((len(chunk), L), self.tok.pad_token_id, dtype=torch.long)
att = torch.zeros((len(chunk), L), dtype=torch.long)
for j, w in enumerate(chunk):
seq = [self.tok.cls_token_id] + w + [self.tok.sep_token_id]
ids[j, :len(seq)] = torch.tensor(seq)
att[j, :len(seq)] = 1
h = self.model.enc(input_ids=ids.to(self.device),
attention_mask=att.to(self.device)).last_hidden_state[:, 0]
cls_vecs.append(h)
return self.model.forward_pieces(torch.cat(cls_vecs),
torch.tensor(owner, device=self.device)).cpu()
def predict(self, message, files, threshold=None):
"""
Predicted class with calibrated probabilities.
Probabilities use the temperature fitted on held-out commits. `margin` is the top
probability minus the runner-up; below `threshold` (default 0.47) the commit is reported
as 'unclassified' rather than forced into a class.
"""
z = self.logits(pieces_from_patches(message, files)) / self.cfg["temperature"]
p = z.softmax(-1)
top2 = p.topk(2).values
margin = float(top2[0] - top2[1])
threshold = self.cfg["margin_threshold"] if threshold is None else threshold
label = self.labels[int(p.argmax())]
return {"label": label if margin >= threshold else "unclassified",
"predicted": label,
"probabilities": {l: round(float(v), 4) for l, v in zip(self.labels, p)},
"margin": round(margin, 4)}
|