Spaces:
Sleeping
Sleeping
File size: 11,676 Bytes
34d3ef3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 | """
app.py - Gradio demo for TechDebt ML (Hugging Face Spaces, ZeroGPU)
Clones a public GitHub repo, runs the same v3 pipeline as src/predict.py
(tabular features + code embeddings) and shows the per-file risk report.
Only the embeddings run on the GPU. If the visitor's ZeroGPU quota is used up,
they are computed on CPU instead (same model, a few seconds slower).
"""
import json
import os
import re
import select
import subprocess
import sys
import tempfile
import time
from pathlib import Path
# src/ lives next to this file on the Space, and one level up in the GitHub repo
HERE = Path(__file__).resolve().parent
for candidate in (HERE / "src", HERE.parent / "src"):
if candidate.is_dir():
sys.path.insert(0, str(candidate))
break
try:
import spaces # must be imported before torch on ZeroGPU
except ImportError: # running locally without the HF `spaces` package
spaces = None
import gradio as gr
import numpy as np
import pandas as pd
import torch
from sentence_transformers import SentenceTransformer
import predict # loads the v3 model, PCA and the CPU embedding model once at startup
from fast_git import git_features
EXAMPLES_DIR = HERE / "examples"
GITHUB_URL_RE = re.compile(r"^https?://github\.com/([\w.-]+)/([\w.-]+?)(?:\.git)?/?$")
MAX_FILES = 150 # larger repos are cut down to their biggest .py files
CLONE_TIMEOUT_S = 90
PER_FILE_TIMEOUT_S = 10 # radon can hang on some huge files
# ZeroGPU wants models placed on cuda at import time; the CPU copy in predict stays as fallback
GPU_EMBED_MODEL = SentenceTransformer("all-MiniLM-L6-v2").to("cuda") if torch.cuda.is_available() else None
# column in the report -> header shown in the UI
SHOWN_COLUMNS = {
"file": "file",
"danger": "risk",
"sloc": "SLOC",
"cyclomatic_complexity": "avg CC",
"max_complexity": "max CC",
"maintainability_index": "MI",
"max_nesting_depth": "nesting",
"num_functions": "funcs",
"commit_count": "commits×",
"author_count": "authors×",
"churn_rate": "churn×",
"days_since_change": "age×",
}
EXAMPLE_REPOS = [
"https://github.com/psf/requests",
"https://github.com/pallets/click",
"https://github.com/Chillipeeper1/techdebt-ml",
]
def parse_github_url(url: str) -> tuple[str, str]:
match = GITHUB_URL_RE.match(url.strip())
if not match:
raise gr.Error("Please paste a public GitHub repo URL, e.g. https://github.com/psf/requests")
return match.group(1), match.group(2)
def clone_repo(owner: str, name: str, dest: Path):
# Full history is required: commit count, authors and churn are model features.
try:
subprocess.run(
["git", "clone", "--quiet", "--single-branch", "--no-tags",
f"https://github.com/{owner}/{name}.git", str(dest)],
check=True, capture_output=True, text=True, timeout=CLONE_TIMEOUT_S,
env={**os.environ, "GIT_TERMINAL_PROMPT": "0"},
)
except subprocess.TimeoutExpired:
raise gr.Error(f"Cloning took more than {CLONE_TIMEOUT_S}s. Try a smaller repo.")
except subprocess.CalledProcessError:
raise gr.Error(f"Could not clone {owner}/{name}. Is it public and does it exist?")
def pick_files(repo_path: Path) -> tuple[list[str], int]:
files = predict.find_python_files(repo_path)
total = len(files)
if total > MAX_FILES:
files = sorted(files, key=lambda f: (repo_path / f).stat().st_size, reverse=True)[:MAX_FILES]
return files, total
def code_metrics(repo_path: Path, files: list[str], progress) -> tuple[dict, list[str]]:
"""Runs metrics_worker.py over the files, killing and restarting it when a
file takes more than PER_FILE_TIMEOUT_S. Returns ({file: output}, skipped)."""
outputs, skipped = {}, []
pending = list(files)
while pending:
proc = subprocess.Popen(
[sys.executable, str(HERE / "metrics_worker.py"), str(repo_path)],
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
)
proc.stdin.write("\n".join(pending).encode())
proc.stdin.close()
buffer = b""
try:
while pending:
if b"\n" not in buffer:
ready, _, _ = select.select([proc.stdout], [], [], PER_FILE_TIMEOUT_S)
chunk = os.read(proc.stdout.fileno(), 1 << 20) if ready else b""
if not chunk: # timed out or worker died on pending[0]
skipped.append(pending.pop(0))
break
buffer += chunk
continue
line, buffer = buffer.split(b"\n", 1)
outputs[pending.pop(0)] = json.loads(line)
progress(0.2 + 0.5 * len(outputs) / len(files), desc=f"Code metrics ({len(outputs)}/{len(files)})")
finally:
proc.kill()
proc.wait()
proc.stdout.close()
return outputs, skipped
def extract_features(repo_path: Path, files: list[str], repo_name: str, progress):
"""Same features as predict.feature_extraction, but with git activity from
fast_git and code metrics from a killable worker process."""
progress(0.1, desc="Reading git history")
git_feats = git_features(repo_path, files)
metrics, skipped = code_metrics(repo_path, files, progress)
results, functions, index = {}, [], []
for file in files:
out = metrics.get(file)
if out is None or "error" in out:
continue
features = {**out["features"], **git_feats[file], "repo": repo_name}
key = f"{repo_name}/{file}".replace("\\", "/")
results[key] = features
index.append((key, len(out["functions"])))
functions.extend(out["functions"])
return results, functions, index, skipped
def file_embeddings(functions: list[str], index: list, model) -> dict:
"""Same as predict.generar_embeddings_reducidos, with the model as a parameter:
encodes every function, then mean-pools them per file (zeros if none)."""
encoded = model.encode(functions, batch_size=64) if functions else np.empty((0, 384))
per_file, cursor = {}, 0
for key, count in index:
per_file[key] = encoded[cursor: cursor + count].mean(axis=0) if count else np.zeros(384)
cursor += count
return per_file
def _embed_on_gpu(functions, index):
return file_embeddings(functions, index, GPU_EMBED_MODEL)
if spaces is not None and GPU_EMBED_MODEL is not None:
_embed_on_gpu = spaces.GPU(duration=30)(_embed_on_gpu)
def embed(functions: list[str], index: list) -> tuple[dict, str]:
if GPU_EMBED_MODEL is not None:
try:
return _embed_on_gpu(functions, index), "GPU"
except Exception as e: # typically the visitor's daily ZeroGPU quota
print(f"GPU embedding failed, falling back to CPU: {e}")
return file_embeddings(functions, index, predict.EMBED_MODEL), "CPU"
def analyze_repo(repo_path: Path, repo_name: str, progress) -> tuple[pd.DataFrame, list[str]]:
files, total = pick_files(repo_path)
if not files:
raise gr.Error("No .py files found. TechDebt ML only supports Python codebases.")
notes = []
if total > len(files):
notes.append(f"The repo has {total} .py files; only the {len(files)} largest were analyzed.")
results, functions, index, skipped = extract_features(repo_path, files, repo_name, progress)
if skipped:
notes.append(f"Skipped {len(skipped)} file(s) that took over {PER_FILE_TIMEOUT_S}s to parse: "
+ ", ".join(f"`{f}`" for f in skipped[:5]))
if not results:
raise gr.Error("Could not extract features from any file.")
progress(0.75, desc=f"Embedding {len(functions)} functions and predicting")
embeddings, device = embed(functions, index)
if device == "CPU" and GPU_EMBED_MODEL is not None:
notes.append("GPU quota used up, embeddings were computed on CPU.")
report = predict.full_report(results, embeddings)
report["file"] = report["file"].str.removeprefix(f"{repo_name}/")
return report, notes
def example_cache_path(owner: str, name: str) -> Path:
return EXAMPLES_DIR / f"{owner}__{name}.csv"
def build_outputs(report: pd.DataFrame, owner: str, name: str, notes: list[str], elapsed: float | None):
high = int((report["danger"] >= 0.5).sum())
summary = [
f"### {owner}/{name}",
f"**{len(report)}** files analyzed · **{high}** with risk ≥ 0.5 · "
f"median risk **{report['danger'].median():.2f}**",
]
if elapsed is None:
summary.append("_Pre-computed example (instant). Any other URL is analyzed live._")
else:
summary.append(f"_Analyzed live in {elapsed:.0f}s._")
summary += [f"- {n}" for n in notes]
top = report.head(15)[["file", "danger"]].iloc[::-1]
csv_path = Path(tempfile.mkdtemp(prefix="techdebt_report_")) / f"techdebt_{owner}_{name}.csv"
report.to_csv(csv_path, index=False)
shown = report[list(SHOWN_COLUMNS)].round(2).rename(columns=SHOWN_COLUMNS)
return "\n\n".join(summary), top, shown, str(csv_path)
def run(url: str, progress=gr.Progress()):
owner, name = parse_github_url(url)
cached = example_cache_path(owner, name)
if cached.exists():
return build_outputs(pd.read_csv(cached), owner, name, [], None)
started = time.monotonic()
with tempfile.TemporaryDirectory(prefix="techdebt_") as tmp:
progress(0, desc=f"Cloning {owner}/{name}")
clone_repo(owner, name, Path(tmp) / name)
report, notes = analyze_repo(Path(tmp) / name, name, progress)
return build_outputs(report, owner, name, notes, time.monotonic() - started)
DESCRIPTION = """
# TechDebt ML
Predicts, **per file**, the probability that a Python file carries technical debt.
It combines code metrics (Radon + AST), the file's git activity relative to its own repo,
and embeddings of its most complex functions (`all-MiniLM-L6-v2`), fed to a Random Forest
trained on 2,503 files from 22 public repos.
Paste a **public GitHub repo** with Python code. Live analysis clones the full history,
so it takes from ~10s to a couple of minutes depending on the repo.
[Source code](https://github.com/Chillipeeper1/techdebt-ml) ·
[Known limitations](https://github.com/Chillipeeper1/techdebt-ml/tree/main/docs/decisions)
"""
with gr.Blocks(title="TechDebt ML") as demo:
gr.Markdown(DESCRIPTION)
with gr.Row():
url = gr.Textbox(label="GitHub repo URL", placeholder="https://github.com/owner/repo", scale=4)
button = gr.Button("Analyze", variant="primary", scale=1)
gr.Examples(EXAMPLE_REPOS, inputs=url, label="Try one (pre-computed, instant)")
summary = gr.Markdown()
chart = gr.BarPlot(x="file", y="danger", label="Top 15 riskiest files",
y_lim=[0, 1], x_label_angle=-45, sort="-y", height=380)
table = gr.Dataframe(label="All analyzed files (highest risk first)")
gr.Markdown(
"CC = cyclomatic complexity · MI = maintainability index · "
"**×** = relative to the repo's median file (2.0 = twice the typical file). "
"The CSV has every feature the model uses."
)
download = gr.File(label="Full report (CSV)")
gr.Markdown(
"A high score is a prompt to look closer, not a verdict. "
"Files in folders named `test` are skipped, same as the CLI."
)
outputs = [summary, chart, table, download]
button.click(run, inputs=url, outputs=outputs)
url.submit(run, inputs=url, outputs=outputs)
if __name__ == "__main__":
demo.queue(default_concurrency_limit=2).launch()
|