codebert-commit-intent / commit_intent.py
Binamra00's picture
Safetensors weights for seeds 42, 1337, 2024; model card, config, inference code
a007f4a verified
Raw History Blame Contribute Delete
7.48 kB
"""
Commit-intent classifier: is a commit corrective, adaptive or perfective?
CodeBERT fine-tuned on Levin & Yehudai's 1,151 labelled commits. It reads the commit message and
the per-file diff, encodes each piece separately, and attention-pools the pieces into one commit
vector. This module rebuilds exactly the architecture that was trained, so the published weights
load into it unchanged.
from commit_intent import CommitIntent
clf = CommitIntent.from_pretrained("Binamra00/codebert-commit-intent")
clf.predict("Fix NPE when the config file is empty",
{"src/main/java/Config.java": patch_text})
Training code and evaluation: github.com/Binamra00/commit-intent-and-refactoring-levels
"""
import json
import re
from pathlib import Path
import torch
import torch.nn as nn
from transformers import AutoConfig, AutoModel, AutoTokenizer
DEFAULT_REPO = "Binamra00/codebert-commit-intent"
HUNK = re.compile(r"^@@ .*? @@")
class CommitClassifier(nn.Module):
"""The trained network. Parameter names match the published state dicts."""
def __init__(self, base_model, n_labels=3, dropout=0.2):
super().__init__()
# Built from the config alone: the fine-tuned encoder weights come from the state dict.
self.enc = AutoModel.from_config(AutoConfig.from_pretrained(base_model))
hid = self.enc.config.hidden_size
self.score = nn.Linear(hid, 1) # attention score per piece
self.drop = nn.Dropout(dropout)
self.out = nn.Linear(hid, n_labels)
def forward_pieces(self, window_cls, owner):
"""window_cls: (windows, hid) [CLS] vectors; owner: piece index of each window."""
n = int(owner.max()) + 1
pv = torch.zeros(n, window_cls.size(1), device=window_cls.device, dtype=window_cls.dtype)
pv.index_add_(0, owner, window_cls)
cnt = torch.bincount(owner, minlength=n).to(window_cls.dtype).unsqueeze(1)
pv = pv / cnt # windows mean-pooled within a piece
a = self.score(pv).softmax(0) # pieces attention-pooled into a commit
return self.out(self.drop((pv * a).sum(0)))
def pieces_from_patches(message, files):
"""
Render a commit the way the model was trained on it.
message : the full commit message, as git records it
files : {path: unified-diff patch} for each changed file -- GitHub's API `patch` field or one
file's section of `git diff`. Header and hunk lines are dropped; '+' lines become
ADDED, '-' lines REMOVED. Context lines are ignored.
Returns the message piece followed by one piece per file, largest first.
"""
pieces = [f"<COMMIT_MESSAGE>{(message or '').strip()}</COMMIT_MESSAGE>"]
rendered = []
for path, patch in dict(files or {}).items():
added, removed = [], []
for ln in (patch or "").split("\n"):
if HUNK.match(ln) or ln.startswith(("--- ", "+++ ")):
continue
if ln.startswith("+"):
added.append(ln[1:])
elif ln.startswith("-"):
removed.append(ln[1:])
part = [f"<FILE>{path}</FILE>"]
if removed:
part.append("<REMOVED>" + "\n".join(removed) + "</REMOVED>")
if added:
part.append("<ADDED>" + "\n".join(added) + "</ADDED>")
rendered.append("\n".join(part))
rendered.sort(key=len, reverse=True)
return pieces + rendered
class CommitIntent:
"""Calibrated predictions from a commit's message and diff."""
def __init__(self, model, tokenizer, config, device):
self.model, self.tok, self.cfg, self.device = model, tokenizer, config, device
self.labels = config["labels"]
@classmethod
def from_pretrained(cls, repo_or_dir=DEFAULT_REPO, seed=None, device=None):
"""Load from a Hugging Face repo id or a local folder holding config.json and the weights."""
from safetensors.torch import load_file
local = Path(repo_or_dir)
if local.is_dir():
fetch = lambda name: str(local / name)
else:
from huggingface_hub import hf_hub_download
fetch = lambda name: hf_hub_download(repo_or_dir, name)
config = json.load(open(fetch("config.json"), encoding="utf-8"))
seed = str(seed or config["default_seed"])
device = device or ("cuda" if torch.cuda.is_available() else "cpu")
model = CommitClassifier(config["base_model"], len(config["labels"]), config["dropout"])
state = load_file(fetch(config["weights"][seed]))
missing, unexpected = model.load_state_dict(state, strict=False)
# Older transformers versions also save a position_ids buffer; nothing else may differ.
stray = [k for k in missing + unexpected if not k.endswith("position_ids")]
if stray:
raise RuntimeError(f"weights do not match the architecture: {stray[:5]}")
model.to(device).eval()
return cls(model, AutoTokenizer.from_pretrained(config["base_model"]), config, device)
def _windows(self, text):
ids = self.tok.encode(text, add_special_tokens=False, truncation=False)
body = self.cfg["max_length"] - 2
return [ids[i:i + body] for i in range(0, max(len(ids), 1), body)] or [[]]
@torch.no_grad()
def logits(self, pieces, batch=64):
"""Raw logits for one commit given its rendered pieces."""
pieces = pieces[: self.cfg["max_pieces"]]
wins, owner = [], []
for pi, p in enumerate(pieces):
for w in self._windows(p):
wins.append(w)
owner.append(pi)
cls_vecs = []
for s in range(0, len(wins), batch):
chunk = wins[s:s + batch]
L = max(len(w) for w in chunk) + 2
ids = torch.full((len(chunk), L), self.tok.pad_token_id, dtype=torch.long)
att = torch.zeros((len(chunk), L), dtype=torch.long)
for j, w in enumerate(chunk):
seq = [self.tok.cls_token_id] + w + [self.tok.sep_token_id]
ids[j, :len(seq)] = torch.tensor(seq)
att[j, :len(seq)] = 1
h = self.model.enc(input_ids=ids.to(self.device),
attention_mask=att.to(self.device)).last_hidden_state[:, 0]
cls_vecs.append(h)
return self.model.forward_pieces(torch.cat(cls_vecs),
torch.tensor(owner, device=self.device)).cpu()
def predict(self, message, files, threshold=None):
"""
Predicted class with calibrated probabilities.
Probabilities use the temperature fitted on held-out commits. `margin` is the top
probability minus the runner-up; below `threshold` (default 0.47) the commit is reported
as 'unclassified' rather than forced into a class.
"""
z = self.logits(pieces_from_patches(message, files)) / self.cfg["temperature"]
p = z.softmax(-1)
top2 = p.topk(2).values
margin = float(top2[0] - top2[1])
threshold = self.cfg["margin_threshold"] if threshold is None else threshold
label = self.labels[int(p.argmax())]
return {"label": label if margin >= threshold else "unclassified",
"predicted": label,
"probabilities": {l: round(float(v), 4) for l, v in zip(self.labels, p)},
"margin": round(margin, 4)}