Spaces:
Sleeping
Sleeping
Download app.py from ChilliPeep/techdebt-ml: direct link, hf CLI and curl.
- Browser
- Download file 11.7 kB
-
https://huggingface.co/spaces/ChilliPeep/techdebt-ml/resolve/main/app.py
- Command line
-
hf download hf://spaces/ChilliPeep/techdebt-ml/app.py
-
curl -L -o app.py https://huggingface.co/spaces/ChilliPeep/techdebt-ml/resolve/main/app.py
11.7 kB
| """ | |
| app.py - Gradio demo for TechDebt ML (Hugging Face Spaces, ZeroGPU) | |
| Clones a public GitHub repo, runs the same v3 pipeline as src/predict.py | |
| (tabular features + code embeddings) and shows the per-file risk report. | |
| Only the embeddings run on the GPU. If the visitor's ZeroGPU quota is used up, | |
| they are computed on CPU instead (same model, a few seconds slower). | |
| """ | |
| import json | |
| import os | |
| import re | |
| import select | |
| import subprocess | |
| import sys | |
| import tempfile | |
| import time | |
| from pathlib import Path | |
| # src/ lives next to this file on the Space, and one level up in the GitHub repo | |
| HERE = Path(__file__).resolve().parent | |
| for candidate in (HERE / "src", HERE.parent / "src"): | |
| if candidate.is_dir(): | |
| sys.path.insert(0, str(candidate)) | |
| break | |
| try: | |
| import spaces # must be imported before torch on ZeroGPU | |
| except ImportError: # running locally without the HF `spaces` package | |
| spaces = None | |
| import gradio as gr | |
| import numpy as np | |
| import pandas as pd | |
| import torch | |
| from sentence_transformers import SentenceTransformer | |
| import predict # loads the v3 model, PCA and the CPU embedding model once at startup | |
| from fast_git import git_features | |
| EXAMPLES_DIR = HERE / "examples" | |
| GITHUB_URL_RE = re.compile(r"^https?://github\.com/([\w.-]+)/([\w.-]+?)(?:\.git)?/?$") | |
| MAX_FILES = 150 # larger repos are cut down to their biggest .py files | |
| CLONE_TIMEOUT_S = 90 | |
| PER_FILE_TIMEOUT_S = 10 # radon can hang on some huge files | |
| # ZeroGPU wants models placed on cuda at import time; the CPU copy in predict stays as fallback | |
| GPU_EMBED_MODEL = SentenceTransformer("all-MiniLM-L6-v2").to("cuda") if torch.cuda.is_available() else None | |
| # column in the report -> header shown in the UI | |
| SHOWN_COLUMNS = { | |
| "file": "file", | |
| "danger": "risk", | |
| "sloc": "SLOC", | |
| "cyclomatic_complexity": "avg CC", | |
| "max_complexity": "max CC", | |
| "maintainability_index": "MI", | |
| "max_nesting_depth": "nesting", | |
| "num_functions": "funcs", | |
| "commit_count": "commits×", | |
| "author_count": "authors×", | |
| "churn_rate": "churn×", | |
| "days_since_change": "age×", | |
| } | |
| EXAMPLE_REPOS = [ | |
| "https://github.com/psf/requests", | |
| "https://github.com/pallets/click", | |
| "https://github.com/Chillipeeper1/techdebt-ml", | |
| ] | |
| def parse_github_url(url: str) -> tuple[str, str]: | |
| match = GITHUB_URL_RE.match(url.strip()) | |
| if not match: | |
| raise gr.Error("Please paste a public GitHub repo URL, e.g. https://github.com/psf/requests") | |
| return match.group(1), match.group(2) | |
| def clone_repo(owner: str, name: str, dest: Path): | |
| # Full history is required: commit count, authors and churn are model features. | |
| try: | |
| subprocess.run( | |
| ["git", "clone", "--quiet", "--single-branch", "--no-tags", | |
| f"https://github.com/{owner}/{name}.git", str(dest)], | |
| check=True, capture_output=True, text=True, timeout=CLONE_TIMEOUT_S, | |
| env={**os.environ, "GIT_TERMINAL_PROMPT": "0"}, | |
| ) | |
| except subprocess.TimeoutExpired: | |
| raise gr.Error(f"Cloning took more than {CLONE_TIMEOUT_S}s. Try a smaller repo.") | |
| except subprocess.CalledProcessError: | |
| raise gr.Error(f"Could not clone {owner}/{name}. Is it public and does it exist?") | |
| def pick_files(repo_path: Path) -> tuple[list[str], int]: | |
| files = predict.find_python_files(repo_path) | |
| total = len(files) | |
| if total > MAX_FILES: | |
| files = sorted(files, key=lambda f: (repo_path / f).stat().st_size, reverse=True)[:MAX_FILES] | |
| return files, total | |
| def code_metrics(repo_path: Path, files: list[str], progress) -> tuple[dict, list[str]]: | |
| """Runs metrics_worker.py over the files, killing and restarting it when a | |
| file takes more than PER_FILE_TIMEOUT_S. Returns ({file: output}, skipped).""" | |
| outputs, skipped = {}, [] | |
| pending = list(files) | |
| while pending: | |
| proc = subprocess.Popen( | |
| [sys.executable, str(HERE / "metrics_worker.py"), str(repo_path)], | |
| stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, | |
| ) | |
| proc.stdin.write("\n".join(pending).encode()) | |
| proc.stdin.close() | |
| buffer = b"" | |
| try: | |
| while pending: | |
| if b"\n" not in buffer: | |
| ready, _, _ = select.select([proc.stdout], [], [], PER_FILE_TIMEOUT_S) | |
| chunk = os.read(proc.stdout.fileno(), 1 << 20) if ready else b"" | |
| if not chunk: # timed out or worker died on pending[0] | |
| skipped.append(pending.pop(0)) | |
| break | |
| buffer += chunk | |
| continue | |
| line, buffer = buffer.split(b"\n", 1) | |
| outputs[pending.pop(0)] = json.loads(line) | |
| progress(0.2 + 0.5 * len(outputs) / len(files), desc=f"Code metrics ({len(outputs)}/{len(files)})") | |
| finally: | |
| proc.kill() | |
| proc.wait() | |
| proc.stdout.close() | |
| return outputs, skipped | |
| def extract_features(repo_path: Path, files: list[str], repo_name: str, progress): | |
| """Same features as predict.feature_extraction, but with git activity from | |
| fast_git and code metrics from a killable worker process.""" | |
| progress(0.1, desc="Reading git history") | |
| git_feats = git_features(repo_path, files) | |
| metrics, skipped = code_metrics(repo_path, files, progress) | |
| results, functions, index = {}, [], [] | |
| for file in files: | |
| out = metrics.get(file) | |
| if out is None or "error" in out: | |
| continue | |
| features = {**out["features"], **git_feats[file], "repo": repo_name} | |
| key = f"{repo_name}/{file}".replace("\\", "/") | |
| results[key] = features | |
| index.append((key, len(out["functions"]))) | |
| functions.extend(out["functions"]) | |
| return results, functions, index, skipped | |
| def file_embeddings(functions: list[str], index: list, model) -> dict: | |
| """Same as predict.generar_embeddings_reducidos, with the model as a parameter: | |
| encodes every function, then mean-pools them per file (zeros if none).""" | |
| encoded = model.encode(functions, batch_size=64) if functions else np.empty((0, 384)) | |
| per_file, cursor = {}, 0 | |
| for key, count in index: | |
| per_file[key] = encoded[cursor: cursor + count].mean(axis=0) if count else np.zeros(384) | |
| cursor += count | |
| return per_file | |
| def _embed_on_gpu(functions, index): | |
| return file_embeddings(functions, index, GPU_EMBED_MODEL) | |
| if spaces is not None and GPU_EMBED_MODEL is not None: | |
| _embed_on_gpu = spaces.GPU(duration=30)(_embed_on_gpu) | |
| def embed(functions: list[str], index: list) -> tuple[dict, str]: | |
| if GPU_EMBED_MODEL is not None: | |
| try: | |
| return _embed_on_gpu(functions, index), "GPU" | |
| except Exception as e: # typically the visitor's daily ZeroGPU quota | |
| print(f"GPU embedding failed, falling back to CPU: {e}") | |
| return file_embeddings(functions, index, predict.EMBED_MODEL), "CPU" | |
| def analyze_repo(repo_path: Path, repo_name: str, progress) -> tuple[pd.DataFrame, list[str]]: | |
| files, total = pick_files(repo_path) | |
| if not files: | |
| raise gr.Error("No .py files found. TechDebt ML only supports Python codebases.") | |
| notes = [] | |
| if total > len(files): | |
| notes.append(f"The repo has {total} .py files; only the {len(files)} largest were analyzed.") | |
| results, functions, index, skipped = extract_features(repo_path, files, repo_name, progress) | |
| if skipped: | |
| notes.append(f"Skipped {len(skipped)} file(s) that took over {PER_FILE_TIMEOUT_S}s to parse: " | |
| + ", ".join(f"`{f}`" for f in skipped[:5])) | |
| if not results: | |
| raise gr.Error("Could not extract features from any file.") | |
| progress(0.75, desc=f"Embedding {len(functions)} functions and predicting") | |
| embeddings, device = embed(functions, index) | |
| if device == "CPU" and GPU_EMBED_MODEL is not None: | |
| notes.append("GPU quota used up, embeddings were computed on CPU.") | |
| report = predict.full_report(results, embeddings) | |
| report["file"] = report["file"].str.removeprefix(f"{repo_name}/") | |
| return report, notes | |
| def example_cache_path(owner: str, name: str) -> Path: | |
| return EXAMPLES_DIR / f"{owner}__{name}.csv" | |
| def build_outputs(report: pd.DataFrame, owner: str, name: str, notes: list[str], elapsed: float | None): | |
| high = int((report["danger"] >= 0.5).sum()) | |
| summary = [ | |
| f"### {owner}/{name}", | |
| f"**{len(report)}** files analyzed · **{high}** with risk ≥ 0.5 · " | |
| f"median risk **{report['danger'].median():.2f}**", | |
| ] | |
| if elapsed is None: | |
| summary.append("_Pre-computed example (instant). Any other URL is analyzed live._") | |
| else: | |
| summary.append(f"_Analyzed live in {elapsed:.0f}s._") | |
| summary += [f"- {n}" for n in notes] | |
| top = report.head(15)[["file", "danger"]].iloc[::-1] | |
| csv_path = Path(tempfile.mkdtemp(prefix="techdebt_report_")) / f"techdebt_{owner}_{name}.csv" | |
| report.to_csv(csv_path, index=False) | |
| shown = report[list(SHOWN_COLUMNS)].round(2).rename(columns=SHOWN_COLUMNS) | |
| return "\n\n".join(summary), top, shown, str(csv_path) | |
| def run(url: str, progress=gr.Progress()): | |
| owner, name = parse_github_url(url) | |
| cached = example_cache_path(owner, name) | |
| if cached.exists(): | |
| return build_outputs(pd.read_csv(cached), owner, name, [], None) | |
| started = time.monotonic() | |
| with tempfile.TemporaryDirectory(prefix="techdebt_") as tmp: | |
| progress(0, desc=f"Cloning {owner}/{name}") | |
| clone_repo(owner, name, Path(tmp) / name) | |
| report, notes = analyze_repo(Path(tmp) / name, name, progress) | |
| return build_outputs(report, owner, name, notes, time.monotonic() - started) | |
| DESCRIPTION = """ | |
| # TechDebt ML | |
| Predicts, **per file**, the probability that a Python file carries technical debt. | |
| It combines code metrics (Radon + AST), the file's git activity relative to its own repo, | |
| and embeddings of its most complex functions (`all-MiniLM-L6-v2`), fed to a Random Forest | |
| trained on 2,503 files from 22 public repos. | |
| Paste a **public GitHub repo** with Python code. Live analysis clones the full history, | |
| so it takes from ~10s to a couple of minutes depending on the repo. | |
| [Source code](https://github.com/Chillipeeper1/techdebt-ml) · | |
| [Known limitations](https://github.com/Chillipeeper1/techdebt-ml/tree/main/docs/decisions) | |
| """ | |
| with gr.Blocks(title="TechDebt ML") as demo: | |
| gr.Markdown(DESCRIPTION) | |
| with gr.Row(): | |
| url = gr.Textbox(label="GitHub repo URL", placeholder="https://github.com/owner/repo", scale=4) | |
| button = gr.Button("Analyze", variant="primary", scale=1) | |
| gr.Examples(EXAMPLE_REPOS, inputs=url, label="Try one (pre-computed, instant)") | |
| summary = gr.Markdown() | |
| chart = gr.BarPlot(x="file", y="danger", label="Top 15 riskiest files", | |
| y_lim=[0, 1], x_label_angle=-45, sort="-y", height=380) | |
| table = gr.Dataframe(label="All analyzed files (highest risk first)") | |
| gr.Markdown( | |
| "CC = cyclomatic complexity · MI = maintainability index · " | |
| "**×** = relative to the repo's median file (2.0 = twice the typical file). " | |
| "The CSV has every feature the model uses." | |
| ) | |
| download = gr.File(label="Full report (CSV)") | |
| gr.Markdown( | |
| "A high score is a prompt to look closer, not a verdict. " | |
| "Files in folders named `test` are skipped, same as the CLI." | |
| ) | |
| outputs = [summary, chart, table, download] | |
| button.click(run, inputs=url, outputs=outputs) | |
| url.submit(run, inputs=url, outputs=outputs) | |
| if __name__ == "__main__": | |
| demo.queue(default_concurrency_limit=2).launch() | |