classifs / quality /infer.py
anshy's picture
Upload folder using huggingface_hub
7f390fb verified
Raw History Blame Contribute Delete
5.58 kB
"""Standalone inference: fetch a live GitHub PR and score it with this quality
classifier. Self-contained -- only needs `requests` and `fasttext` installed
(see /ansh/fasttext-stuff/requirements.txt: fasttext==0.9.3, numpy<2, scipy<1.14
if you also want the correlation-metric scripts; numpy<2 alone is required for
fasttext's batch predict() to work at all).
Usage:
export GITHUB_PAT=... # a GitHub personal access token (read-only is fine)
python infer.py https://github.com/owner/repo/pull/123 [more PR urls...]
The text fed to the model is built EXACTLY the way training data was: the same
render_document() + fasttext_preprocess() pipeline as annotate_pr_quality.py, so a
score here is directly comparable to what's in train.txt/val.txt.
"""
import base64
import os
import re
import sys
import fasttext
import requests
HERE = os.path.dirname(os.path.abspath(__file__))
MODEL_PATH = os.path.join(HERE, "model.bin")
MAX_FILES = 10
MAX_FILE_CHARS = 4_000
MAX_DIFF_CHARS = 12_000
MAX_FASTTEXT_TOKENS = 4_096
NON_CODE_EXTS = {"json", "xml"}
TOKEN_PATTERN = re.compile(r"\w+(?:['’]\w+)*|[^\w\s]", re.UNICODE)
TOKEN = os.environ["GITHUB_PAT"]
HEADERS = {"Authorization": f"token {TOKEN}", "Accept": "application/vnd.github+json"}
def truncate(text, limit):
if text is None:
return None
if len(text) <= limit:
return text
return text[:limit] + f"\n...[truncated, {len(text) - limit} more chars]"
def fasttext_preprocess(text):
tokens = []
for match in TOKEN_PATTERN.finditer(text.casefold()):
if len(tokens) == MAX_FASTTEXT_TOKENS:
break
token = match.group()
if token.startswith("__label__"):
token = "literal_" + token
tokens.append(token)
return " ".join(tokens)
def render_document(state):
lines = [f"repo : {state['repo']['full_name']}"]
if state["repo"]["description"]:
lines.append(f"description : {state['repo']['description']}")
lines.append(f"stars : {state['repo']['stars']}")
if state["repo"]["license"]:
lines.append(f"license : {state['repo']['license']}")
if state["pull_request_title"]:
lines.append(f"title : {state['pull_request_title']}")
lines.append("touched files : " + ", ".join(state["all_touched_files"]))
for f in state["files"]:
lines.append(f"--- {f['path']} (pre-change) ---")
lines.append(f["pre_change_content"] or "[new file]")
lines.append("--- diff ---")
lines.append(state["diff"] or "")
return "\n".join(lines)
def get(url, headers=None, **kw):
resp = requests.get(url, headers=headers or HEADERS, timeout=20, **kw)
resp.raise_for_status()
return resp
def parse_pr_url(url):
m = re.search(r"github\.com/([^/]+)/([^/]+)/pull/(\d+)", url)
return m.group(1), m.group(2), int(m.group(3))
def fetch_state(pr_url):
owner, repo, number = parse_pr_url(pr_url)
repo_full = f"{owner}/{repo}"
repo_info = get(f"https://api.github.com/repos/{repo_full}").json()
pr_info = get(f"https://api.github.com/repos/{repo_full}/pulls/{number}").json()
files_info = get(f"https://api.github.com/repos/{repo_full}/pulls/{number}/files",
params={"per_page": 100}).json()
diff = get(f"https://api.github.com/repos/{repo_full}/pulls/{number}",
headers={**HEADERS, "Accept": "application/vnd.github.v3.diff"}).text
base_sha = pr_info["base"]["sha"]
all_touched_files = [f["filename"] for f in files_info]
picked = []
for f in files_info:
path = f["filename"]
ext = path.rsplit(".", 1)[-1].lower() if "." in path else ""
if ext in NON_CODE_EXTS:
continue
content = None
if f["status"] != "added":
try:
c = get(f"https://api.github.com/repos/{repo_full}/contents/{path}",
params={"ref": base_sha}).json()
if c.get("encoding") == "base64":
content = base64.b64decode(c["content"]).decode("utf-8", errors="replace")
except requests.HTTPError:
content = None
picked.append({"path": path, "pre_change_content": truncate(content, MAX_FILE_CHARS)})
if len(picked) == MAX_FILES:
break
license_ = repo_info.get("license") or {}
return {
"repo": {
"full_name": repo_full,
"description": repo_info.get("description"),
"stars": repo_info.get("stargazers_count"),
"license": license_.get("spdx_id") or license_.get("name"),
},
"pull_request_title": pr_info.get("title"),
"all_touched_files": all_touched_files,
"files": picked,
"diff": truncate(diff, MAX_DIFF_CHARS),
}
def predict(model, text):
n_labels = len(model.get_labels())
labels, probs = model.predict(text, k=n_labels)
soft = sum(p * int(l.rsplit("_", 1)[1]) for l, p in zip(labels, probs))
hard = int(labels[0].rsplit("_", 1)[1])
return soft, hard
if __name__ == "__main__":
if len(sys.argv) < 2:
raise SystemExit("usage: python infer.py <github PR url> [more PR urls...]")
model = fasttext.load_model(MODEL_PATH)
for pr_url in sys.argv[1:]:
state = fetch_state(pr_url)
text = fasttext_preprocess(render_document(state))
soft, hard = predict(model, text)
print(f"{pr_url}")
print(f" repo: {state['repo']['full_name']} stars: {state['repo']['stars']}")
print(f" predicted score: soft={soft:.2f} hard_level={hard}\n")