Spaces:
Sleeping
Sleeping
Deploy demo
Browse files- README.md +18 -6
- app.py +295 -0
- examples/Chillipeeper1__techdebt-ml.csv +10 -0
- examples/pallets__click.csv +92 -0
- examples/psf__requests.csv +38 -0
- fast_git.py +77 -0
- metrics_worker.py +40 -0
- models/features_v3.pkl +3 -0
- models/pca_v3.pkl +3 -0
- models/random_forest_v3.pkl +3 -0
- requirements.txt +10 -0
- src/feature_lib.py +190 -0
- src/predict.py +281 -0
README.md
CHANGED
|
@@ -1,13 +1,25 @@
|
|
| 1 |
---
|
| 2 |
-
title:
|
| 3 |
-
emoji:
|
| 4 |
colorFrom: yellow
|
| 5 |
-
colorTo:
|
| 6 |
sdk: gradio
|
| 7 |
-
sdk_version: 6.
|
| 8 |
-
python_version:
|
| 9 |
app_file: app.py
|
| 10 |
pinned: false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
---
|
| 12 |
|
| 13 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: TechDebt ML
|
| 3 |
+
emoji: 🧹
|
| 4 |
colorFrom: yellow
|
| 5 |
+
colorTo: red
|
| 6 |
sdk: gradio
|
| 7 |
+
sdk_version: 6.29.1
|
| 8 |
+
python_version: "3.12"
|
| 9 |
app_file: app.py
|
| 10 |
pinned: false
|
| 11 |
+
license: mit
|
| 12 |
+
short_description: Predicts technical debt risk per file in Python repos
|
| 13 |
+
models:
|
| 14 |
+
- sentence-transformers/all-MiniLM-L6-v2
|
| 15 |
+
datasets:
|
| 16 |
+
- ChilliPeep/techdebt-ml-dataset
|
| 17 |
---
|
| 18 |
|
| 19 |
+
# TechDebt ML — demo
|
| 20 |
+
|
| 21 |
+
Paste a public GitHub repo with Python code and get, per file, the predicted probability
|
| 22 |
+
that it carries technical debt.
|
| 23 |
+
|
| 24 |
+
Source code, training notebook and known limitations:
|
| 25 |
+
[github.com/Chillipeeper1/techdebt-ml](https://github.com/Chillipeeper1/techdebt-ml)
|
app.py
ADDED
|
@@ -0,0 +1,295 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
app.py - Gradio demo for TechDebt ML (Hugging Face Spaces, ZeroGPU)
|
| 3 |
+
|
| 4 |
+
Clones a public GitHub repo, runs the same v3 pipeline as src/predict.py
|
| 5 |
+
(tabular features + code embeddings) and shows the per-file risk report.
|
| 6 |
+
|
| 7 |
+
Only the embeddings run on the GPU. If the visitor's ZeroGPU quota is used up,
|
| 8 |
+
they are computed on CPU instead (same model, a few seconds slower).
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
import json
|
| 12 |
+
import os
|
| 13 |
+
import re
|
| 14 |
+
import select
|
| 15 |
+
import subprocess
|
| 16 |
+
import sys
|
| 17 |
+
import tempfile
|
| 18 |
+
import time
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
|
| 21 |
+
# src/ lives next to this file on the Space, and one level up in the GitHub repo
|
| 22 |
+
HERE = Path(__file__).resolve().parent
|
| 23 |
+
for candidate in (HERE / "src", HERE.parent / "src"):
|
| 24 |
+
if candidate.is_dir():
|
| 25 |
+
sys.path.insert(0, str(candidate))
|
| 26 |
+
break
|
| 27 |
+
|
| 28 |
+
try:
|
| 29 |
+
import spaces # must be imported before torch on ZeroGPU
|
| 30 |
+
except ImportError: # running locally without the HF `spaces` package
|
| 31 |
+
spaces = None
|
| 32 |
+
|
| 33 |
+
import gradio as gr
|
| 34 |
+
import numpy as np
|
| 35 |
+
import pandas as pd
|
| 36 |
+
import torch
|
| 37 |
+
from sentence_transformers import SentenceTransformer
|
| 38 |
+
|
| 39 |
+
import predict # loads the v3 model, PCA and the CPU embedding model once at startup
|
| 40 |
+
from fast_git import git_features
|
| 41 |
+
|
| 42 |
+
EXAMPLES_DIR = HERE / "examples"
|
| 43 |
+
GITHUB_URL_RE = re.compile(r"^https?://github\.com/([\w.-]+)/([\w.-]+?)(?:\.git)?/?$")
|
| 44 |
+
|
| 45 |
+
MAX_FILES = 150 # larger repos are cut down to their biggest .py files
|
| 46 |
+
CLONE_TIMEOUT_S = 90
|
| 47 |
+
PER_FILE_TIMEOUT_S = 10 # radon can hang on some huge files
|
| 48 |
+
|
| 49 |
+
# ZeroGPU wants models placed on cuda at import time; the CPU copy in predict stays as fallback
|
| 50 |
+
GPU_EMBED_MODEL = SentenceTransformer("all-MiniLM-L6-v2").to("cuda") if torch.cuda.is_available() else None
|
| 51 |
+
|
| 52 |
+
# column in the report -> header shown in the UI
|
| 53 |
+
SHOWN_COLUMNS = {
|
| 54 |
+
"file": "file",
|
| 55 |
+
"danger": "risk",
|
| 56 |
+
"sloc": "SLOC",
|
| 57 |
+
"cyclomatic_complexity": "avg CC",
|
| 58 |
+
"max_complexity": "max CC",
|
| 59 |
+
"maintainability_index": "MI",
|
| 60 |
+
"max_nesting_depth": "nesting",
|
| 61 |
+
"num_functions": "funcs",
|
| 62 |
+
"commit_count": "commits×",
|
| 63 |
+
"author_count": "authors×",
|
| 64 |
+
"churn_rate": "churn×",
|
| 65 |
+
"days_since_change": "age×",
|
| 66 |
+
}
|
| 67 |
+
|
| 68 |
+
EXAMPLE_REPOS = [
|
| 69 |
+
"https://github.com/psf/requests",
|
| 70 |
+
"https://github.com/pallets/click",
|
| 71 |
+
"https://github.com/Chillipeeper1/techdebt-ml",
|
| 72 |
+
]
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def parse_github_url(url: str) -> tuple[str, str]:
|
| 76 |
+
match = GITHUB_URL_RE.match(url.strip())
|
| 77 |
+
if not match:
|
| 78 |
+
raise gr.Error("Please paste a public GitHub repo URL, e.g. https://github.com/psf/requests")
|
| 79 |
+
return match.group(1), match.group(2)
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def clone_repo(owner: str, name: str, dest: Path):
|
| 83 |
+
# Full history is required: commit count, authors and churn are model features.
|
| 84 |
+
try:
|
| 85 |
+
subprocess.run(
|
| 86 |
+
["git", "clone", "--quiet", "--single-branch", "--no-tags",
|
| 87 |
+
f"https://github.com/{owner}/{name}.git", str(dest)],
|
| 88 |
+
check=True, capture_output=True, text=True, timeout=CLONE_TIMEOUT_S,
|
| 89 |
+
env={**os.environ, "GIT_TERMINAL_PROMPT": "0"},
|
| 90 |
+
)
|
| 91 |
+
except subprocess.TimeoutExpired:
|
| 92 |
+
raise gr.Error(f"Cloning took more than {CLONE_TIMEOUT_S}s. Try a smaller repo.")
|
| 93 |
+
except subprocess.CalledProcessError:
|
| 94 |
+
raise gr.Error(f"Could not clone {owner}/{name}. Is it public and does it exist?")
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def pick_files(repo_path: Path) -> tuple[list[str], int]:
|
| 98 |
+
files = predict.find_python_files(repo_path)
|
| 99 |
+
total = len(files)
|
| 100 |
+
if total > MAX_FILES:
|
| 101 |
+
files = sorted(files, key=lambda f: (repo_path / f).stat().st_size, reverse=True)[:MAX_FILES]
|
| 102 |
+
return files, total
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def code_metrics(repo_path: Path, files: list[str], progress) -> tuple[dict, list[str]]:
|
| 106 |
+
"""Runs metrics_worker.py over the files, killing and restarting it when a
|
| 107 |
+
file takes more than PER_FILE_TIMEOUT_S. Returns ({file: output}, skipped)."""
|
| 108 |
+
outputs, skipped = {}, []
|
| 109 |
+
pending = list(files)
|
| 110 |
+
while pending:
|
| 111 |
+
proc = subprocess.Popen(
|
| 112 |
+
[sys.executable, str(HERE / "metrics_worker.py"), str(repo_path)],
|
| 113 |
+
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
|
| 114 |
+
)
|
| 115 |
+
proc.stdin.write("\n".join(pending).encode())
|
| 116 |
+
proc.stdin.close()
|
| 117 |
+
buffer = b""
|
| 118 |
+
try:
|
| 119 |
+
while pending:
|
| 120 |
+
if b"\n" not in buffer:
|
| 121 |
+
ready, _, _ = select.select([proc.stdout], [], [], PER_FILE_TIMEOUT_S)
|
| 122 |
+
chunk = os.read(proc.stdout.fileno(), 1 << 20) if ready else b""
|
| 123 |
+
if not chunk: # timed out or worker died on pending[0]
|
| 124 |
+
skipped.append(pending.pop(0))
|
| 125 |
+
break
|
| 126 |
+
buffer += chunk
|
| 127 |
+
continue
|
| 128 |
+
line, buffer = buffer.split(b"\n", 1)
|
| 129 |
+
outputs[pending.pop(0)] = json.loads(line)
|
| 130 |
+
progress(0.2 + 0.5 * len(outputs) / len(files), desc=f"Code metrics ({len(outputs)}/{len(files)})")
|
| 131 |
+
finally:
|
| 132 |
+
proc.kill()
|
| 133 |
+
proc.wait()
|
| 134 |
+
proc.stdout.close()
|
| 135 |
+
return outputs, skipped
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def extract_features(repo_path: Path, files: list[str], repo_name: str, progress):
|
| 139 |
+
"""Same features as predict.feature_extraction, but with git activity from
|
| 140 |
+
fast_git and code metrics from a killable worker process."""
|
| 141 |
+
progress(0.1, desc="Reading git history")
|
| 142 |
+
git_feats = git_features(repo_path, files)
|
| 143 |
+
metrics, skipped = code_metrics(repo_path, files, progress)
|
| 144 |
+
|
| 145 |
+
results, functions, index = {}, [], []
|
| 146 |
+
for file in files:
|
| 147 |
+
out = metrics.get(file)
|
| 148 |
+
if out is None or "error" in out:
|
| 149 |
+
continue
|
| 150 |
+
features = {**out["features"], **git_feats[file], "repo": repo_name}
|
| 151 |
+
key = f"{repo_name}/{file}".replace("\\", "/")
|
| 152 |
+
results[key] = features
|
| 153 |
+
index.append((key, len(out["functions"])))
|
| 154 |
+
functions.extend(out["functions"])
|
| 155 |
+
return results, functions, index, skipped
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def file_embeddings(functions: list[str], index: list, model) -> dict:
|
| 159 |
+
"""Same as predict.generar_embeddings_reducidos, with the model as a parameter:
|
| 160 |
+
encodes every function, then mean-pools them per file (zeros if none)."""
|
| 161 |
+
encoded = model.encode(functions, batch_size=64) if functions else np.empty((0, 384))
|
| 162 |
+
per_file, cursor = {}, 0
|
| 163 |
+
for key, count in index:
|
| 164 |
+
per_file[key] = encoded[cursor: cursor + count].mean(axis=0) if count else np.zeros(384)
|
| 165 |
+
cursor += count
|
| 166 |
+
return per_file
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
def _embed_on_gpu(functions, index):
|
| 170 |
+
return file_embeddings(functions, index, GPU_EMBED_MODEL)
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
if spaces is not None and GPU_EMBED_MODEL is not None:
|
| 174 |
+
_embed_on_gpu = spaces.GPU(duration=30)(_embed_on_gpu)
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
def embed(functions: list[str], index: list) -> tuple[dict, str]:
|
| 178 |
+
if GPU_EMBED_MODEL is not None:
|
| 179 |
+
try:
|
| 180 |
+
return _embed_on_gpu(functions, index), "GPU"
|
| 181 |
+
except Exception as e: # typically the visitor's daily ZeroGPU quota
|
| 182 |
+
print(f"GPU embedding failed, falling back to CPU: {e}")
|
| 183 |
+
return file_embeddings(functions, index, predict.EMBED_MODEL), "CPU"
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
def analyze_repo(repo_path: Path, repo_name: str, progress) -> tuple[pd.DataFrame, list[str]]:
|
| 187 |
+
files, total = pick_files(repo_path)
|
| 188 |
+
if not files:
|
| 189 |
+
raise gr.Error("No .py files found. TechDebt ML only supports Python codebases.")
|
| 190 |
+
|
| 191 |
+
notes = []
|
| 192 |
+
if total > len(files):
|
| 193 |
+
notes.append(f"The repo has {total} .py files; only the {len(files)} largest were analyzed.")
|
| 194 |
+
|
| 195 |
+
results, functions, index, skipped = extract_features(repo_path, files, repo_name, progress)
|
| 196 |
+
if skipped:
|
| 197 |
+
notes.append(f"Skipped {len(skipped)} file(s) that took over {PER_FILE_TIMEOUT_S}s to parse: "
|
| 198 |
+
+ ", ".join(f"`{f}`" for f in skipped[:5]))
|
| 199 |
+
if not results:
|
| 200 |
+
raise gr.Error("Could not extract features from any file.")
|
| 201 |
+
|
| 202 |
+
progress(0.75, desc=f"Embedding {len(functions)} functions and predicting")
|
| 203 |
+
embeddings, device = embed(functions, index)
|
| 204 |
+
if device == "CPU" and GPU_EMBED_MODEL is not None:
|
| 205 |
+
notes.append("GPU quota used up, embeddings were computed on CPU.")
|
| 206 |
+
report = predict.full_report(results, embeddings)
|
| 207 |
+
report["file"] = report["file"].str.removeprefix(f"{repo_name}/")
|
| 208 |
+
return report, notes
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
def example_cache_path(owner: str, name: str) -> Path:
|
| 212 |
+
return EXAMPLES_DIR / f"{owner}__{name}.csv"
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
def build_outputs(report: pd.DataFrame, owner: str, name: str, notes: list[str], elapsed: float | None):
|
| 216 |
+
high = int((report["danger"] >= 0.5).sum())
|
| 217 |
+
summary = [
|
| 218 |
+
f"### {owner}/{name}",
|
| 219 |
+
f"**{len(report)}** files analyzed · **{high}** with risk ≥ 0.5 · "
|
| 220 |
+
f"median risk **{report['danger'].median():.2f}**",
|
| 221 |
+
]
|
| 222 |
+
if elapsed is None:
|
| 223 |
+
summary.append("_Pre-computed example (instant). Any other URL is analyzed live._")
|
| 224 |
+
else:
|
| 225 |
+
summary.append(f"_Analyzed live in {elapsed:.0f}s._")
|
| 226 |
+
summary += [f"- {n}" for n in notes]
|
| 227 |
+
|
| 228 |
+
top = report.head(15)[["file", "danger"]].iloc[::-1]
|
| 229 |
+
|
| 230 |
+
csv_path = Path(tempfile.mkdtemp(prefix="techdebt_report_")) / f"techdebt_{owner}_{name}.csv"
|
| 231 |
+
report.to_csv(csv_path, index=False)
|
| 232 |
+
|
| 233 |
+
shown = report[list(SHOWN_COLUMNS)].round(2).rename(columns=SHOWN_COLUMNS)
|
| 234 |
+
return "\n\n".join(summary), top, shown, str(csv_path)
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
def run(url: str, progress=gr.Progress()):
|
| 238 |
+
owner, name = parse_github_url(url)
|
| 239 |
+
|
| 240 |
+
cached = example_cache_path(owner, name)
|
| 241 |
+
if cached.exists():
|
| 242 |
+
return build_outputs(pd.read_csv(cached), owner, name, [], None)
|
| 243 |
+
|
| 244 |
+
started = time.monotonic()
|
| 245 |
+
with tempfile.TemporaryDirectory(prefix="techdebt_") as tmp:
|
| 246 |
+
progress(0, desc=f"Cloning {owner}/{name}")
|
| 247 |
+
clone_repo(owner, name, Path(tmp) / name)
|
| 248 |
+
report, notes = analyze_repo(Path(tmp) / name, name, progress)
|
| 249 |
+
|
| 250 |
+
return build_outputs(report, owner, name, notes, time.monotonic() - started)
|
| 251 |
+
|
| 252 |
+
|
| 253 |
+
DESCRIPTION = """
|
| 254 |
+
# TechDebt ML
|
| 255 |
+
|
| 256 |
+
Predicts, **per file**, the probability that a Python file carries technical debt.
|
| 257 |
+
It combines code metrics (Radon + AST), the file's git activity relative to its own repo,
|
| 258 |
+
and embeddings of its most complex functions (`all-MiniLM-L6-v2`), fed to a Random Forest
|
| 259 |
+
trained on 2,503 files from 22 public repos.
|
| 260 |
+
|
| 261 |
+
Paste a **public GitHub repo** with Python code. Live analysis clones the full history,
|
| 262 |
+
so it takes from ~10s to a couple of minutes depending on the repo.
|
| 263 |
+
[Source code](https://github.com/Chillipeeper1/techdebt-ml) ·
|
| 264 |
+
[Known limitations](https://github.com/Chillipeeper1/techdebt-ml/tree/main/docs/decisions)
|
| 265 |
+
"""
|
| 266 |
+
|
| 267 |
+
with gr.Blocks(title="TechDebt ML") as demo:
|
| 268 |
+
gr.Markdown(DESCRIPTION)
|
| 269 |
+
with gr.Row():
|
| 270 |
+
url = gr.Textbox(label="GitHub repo URL", placeholder="https://github.com/owner/repo", scale=4)
|
| 271 |
+
button = gr.Button("Analyze", variant="primary", scale=1)
|
| 272 |
+
gr.Examples(EXAMPLE_REPOS, inputs=url, label="Try one (pre-computed, instant)")
|
| 273 |
+
|
| 274 |
+
summary = gr.Markdown()
|
| 275 |
+
chart = gr.BarPlot(x="file", y="danger", label="Top 15 riskiest files",
|
| 276 |
+
y_lim=[0, 1], x_label_angle=-45, sort="-y", height=380)
|
| 277 |
+
table = gr.Dataframe(label="All analyzed files (highest risk first)")
|
| 278 |
+
gr.Markdown(
|
| 279 |
+
"CC = cyclomatic complexity · MI = maintainability index · "
|
| 280 |
+
"**×** = relative to the repo's median file (2.0 = twice the typical file). "
|
| 281 |
+
"The CSV has every feature the model uses."
|
| 282 |
+
)
|
| 283 |
+
download = gr.File(label="Full report (CSV)")
|
| 284 |
+
|
| 285 |
+
gr.Markdown(
|
| 286 |
+
"A high score is a prompt to look closer, not a verdict. "
|
| 287 |
+
"Files in folders named `test` are skipped, same as the CLI."
|
| 288 |
+
)
|
| 289 |
+
|
| 290 |
+
outputs = [summary, chart, table, download]
|
| 291 |
+
button.click(run, inputs=url, outputs=outputs)
|
| 292 |
+
url.submit(run, inputs=url, outputs=outputs)
|
| 293 |
+
|
| 294 |
+
if __name__ == "__main__":
|
| 295 |
+
demo.queue(default_concurrency_limit=2).launch()
|
examples/Chillipeeper1__techdebt-ml.csv
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
file,danger,loc,sloc,comment_ratio,blank_ratio,cyclomatic_complexity,max_complexity,maintainability_index,halstead_effort,num_functions,avg_function_len,max_nesting_depth,is_tested,dependency_count,commit_count,author_count,days_since_change,churn_rate,repo
|
| 2 |
+
src/predict.py,0.57,281,205,0.046263345195729534,0.2277580071174377,3.6,7,41.39601673862559,4930.789048557579,10,20.8,4,0,17,1.5,1.0,0.9746835443037974,2.890625,techdebt-ml
|
| 3 |
+
src/repo_search.py,0.52,292,195,0.09931506849315068,0.2328767123287671,3.111111111111111,6,51.947356266687024,3574.2223636474364,9,24.22222222222222,3,0,7,1.5,1.0,1.0,2.6875,techdebt-ml
|
| 4 |
+
tests/test_feature_lib.py,0.45,160,112,0.025,0.29375,3.0,5,48.57537705455356,1728.7502218695658,12,2.5,1,1,5,1.0,1.0,1.0,1.5625,techdebt-ml
|
| 5 |
+
src/feature_extraction.py,0.4,78,63,0.14102564102564102,0.08974358974358974,17.0,17,64.11771733993977,1282.0406359950857,1,70.0,6,0,5,1.5,1.0,1.0,2.5520833333333335,techdebt-ml
|
| 6 |
+
src/source_repos.py,0.25,15,11,0.0,0.26666666666666666,0.0,0,63.9732329289583,82.49268997791475,0,0.0,0,0,2,0.5,1.0,1.0,0.078125,techdebt-ml
|
| 7 |
+
src/feature_lib.py,0.19,190,152,0.07894736842105263,0.17894736842105263,2.4,6,47.82040322245754,2994.7935319735184,15,15.333333333333334,3,0,5,1.0,1.0,0.9746835443037974,1.0,techdebt-ml
|
| 8 |
+
src/file_embedding.py,0.17,153,112,0.0,0.2679738562091503,0.0,0,35.91452980437473,1721.5407704102943,0,0.0,3,0,9,0.5,1.0,0.9746835443037974,0.796875,techdebt-ml
|
| 9 |
+
src/hf_upload.py,0.15,27,25,0.0,0.07407407407407407,0.0,0,65.3795898712936,35.19781364344408,0,0.0,0,0,2,0.5,1.0,1.0,0.140625,techdebt-ml
|
| 10 |
+
src/embedding_test.py,0.08,32,27,0.0,0.15625,1.0,1,67.2293414982706,15.509775004326936,1,2.0,1,0,2,0.5,1.0,0.9746835443037974,0.16666666666666666,techdebt-ml
|
examples/pallets__click.csv
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
file,danger,loc,sloc,comment_ratio,blank_ratio,cyclomatic_complexity,max_complexity,maintainability_index,halstead_effort,num_functions,avg_function_len,max_nesting_depth,is_tested,dependency_count,commit_count,author_count,days_since_change,churn_rate,repo
|
| 2 |
+
tests/test_compat.py,0.99,270,225,0.13703703703703704,0.12222222222222222,3.0833333333333335,4,58.38203460551011,622.505329527493,12,1.0,2,1,9,4.0,2.6666666666666665,1.0,3.472,click
|
| 3 |
+
src/click/shell_completion.py,0.96,801,454,0.22971285892634208,0.19850187265917604,2.575,10,43.57881522831033,7886.297733279147,40,19.175,6,0,18,13.75,10.0,0.8387096774193549,12.904,click
|
| 4 |
+
src/click/_winconsole.py,0.96,292,223,0.06164383561643835,0.1917808219178082,2.16,7,43.50839835299815,2904.565396356504,25,10.56,4,0,29,5.75,3.0,0.43010752688172044,9.488,click
|
| 5 |
+
src/click/_compat.py,0.95,590,411,0.09830508474576272,0.2,2.510204081632653,21,32.41820208606701,7642.443443862683,49,11.755102040816327,4,0,14,10.5,4.0,0.5806451612903226,23.784,click
|
| 6 |
+
src/click/decorators.py,0.95,627,300,0.2966507177033493,0.22966507177033493,2.1363636363636362,10,52.746122309232156,5263.344085907805,22,24.272727272727273,6,0,15,14.25,6.333333333333333,1.021505376344086,14.688,click
|
| 7 |
+
tests/test_termui.py,0.95,1984,1477,0.1159274193548387,0.17036290322580644,2.6530612244897958,17,10.785013892429287,24431.874907066154,98,5.091836734693878,4,1,21,16.75,8.666666666666666,0.5591397849462365,25.416,click
|
| 8 |
+
src/click/testing.py,0.95,798,451,0.2731829573934837,0.17794486215538846,2.3902439024390243,19,40.971086579830434,6074.847468633201,41,31.804878048780488,6,0,20,12.0,6.666666666666667,0.8817204301075269,13.432,click
|
| 9 |
+
tests/test_shell_completion.py,0.94,617,482,0.05348460291734198,0.16531604538087522,3.5714285714285716,9,23.28086307554391,9656.210246782257,42,1.6428571428571428,2,1,19,7.25,6.666666666666667,0.8817204301075269,7.824,click
|
| 10 |
+
tests/test_options.py,0.94,3945,3217,0.10266159695817491,0.13079847908745248,3.5970149253731343,13,0.0,50585.12683044013,134,3.425373134328358,4,1,16,34.75,17.333333333333332,0.1827956989247312,49.52,click
|
| 11 |
+
src/click/utils.py,0.94,676,317,0.3210059171597633,0.21005917159763313,2.764705882352941,12,45.35208196896837,6235.585301901369,34,20.941176470588236,4,0,26,15.0,7.333333333333333,0.3118279569892473,12.656,click
|
| 12 |
+
src/click/termui.py,0.92,1024,447,0.384765625,0.177734375,4.148148148148148,21,41.19924554476005,7104.835440791835,27,33.2962962962963,5,0,31,20.25,11.666666666666666,0.46236559139784944,22.784,click
|
| 13 |
+
src/click/types.py,0.92,1437,823,0.22894919972164232,0.20250521920668058,2.6116504854368934,18,16.379594512263477,18489.539878910222,103,19.54368932038835,6,0,26,21.75,11.333333333333334,0.34408602150537637,31.64,click
|
| 14 |
+
tests/test_testing.py,0.91,840,571,0.08333333333333333,0.23095238095238096,4.368421052631579,15,18.052590520986374,11612.581927998759,38,3.4473684210526314,4,1,14,11.75,8.0,0.8817204301075269,9.728,click
|
| 15 |
+
src/click/_termui_impl.py,0.91,972,610,0.2037037037037037,0.1697530864197531,3.978723404255319,19,19.87412789579279,39746.57621505823,47,28.23404255319149,6,0,38,17.5,9.0,0.5591397849462365,26.088,click
|
| 16 |
+
tests/test_arguments.py,0.9,1229,920,0.06265256305939788,0.19772172497965826,3.4098360655737703,8,10.38871981322617,20350.007984101718,61,2.639344262295082,4,1,9,16.0,6.666666666666667,0.1827956989247312,15.552,click
|
| 17 |
+
tests/test_commands.py,0.9,700,538,0.06428571428571428,0.19142857142857142,3.962962962962963,26,27.58020402791504,6429.701108987239,27,2.185185185185185,4,1,4,11.5,6.666666666666667,0.7634408602150538,10.832,click
|
| 18 |
+
tests/test_basic.py,0.89,986,733,0.0486815415821501,0.21196754563894524,4.476190476190476,11,10.295207289577604,24869.389698705712,42,2.0476190476190474,3,1,9,14.0,6.0,0.3763440860215054,16.8,click
|
| 19 |
+
src/click/core.py,0.88,4038,1975,0.33407627538385337,0.17855373947498762,4.22360248447205,35,0.0,139100.5450469338,161,43.192546583850934,10,0,64,64.75,25.666666666666668,0.1827956989247312,112.72,click
|
| 20 |
+
src/click/exceptions.py,0.87,378,229,0.1455026455026455,0.24074074074074073,2.657142857142857,13,46.9906608001246,4580.507679657695,35,14.714285714285714,5,0,14,9.0,5.333333333333333,1.4838709677419355,6.912,click
|
| 21 |
+
src/click/parser.py,0.84,533,331,0.22889305816135083,0.16135084427767354,4.375,10,40.79188795676464,11417.608584709487,24,32.666666666666664,7,0,20,7.5,4.666666666666667,1.6666666666666667,12.584,click
|
| 22 |
+
tests/test_formatting.py,0.84,632,472,0.1518987341772152,0.13765822784810128,2.9565217391304346,8,45.653005905441425,2794.6177056740407,23,2.130434782608696,3,1,3,8.0,6.0,1.0,8.528,click
|
| 23 |
+
src/click/__init__.py,0.83,144,121,0.0,0.11805555555555555,7.0,7,57.005748291254356,46.285714285714285,1,69.0,2,0,72,11.75,2.6666666666666665,0.9139784946236559,5.248,click
|
| 24 |
+
src/click/formatting.py,0.83,320,197,0.1875,0.175,2.6470588235294117,7,51.91024993233019,4989.098430337277,17,27.0,5,0,8,5.5,3.0,1.4838709677419355,4.624,click
|
| 25 |
+
tests/test_defaults.py,0.83,567,363,0.19576719576719576,0.19753086419753085,4.388888888888889,8,43.09415859790043,7408.418224384461,18,1.1666666666666667,4,1,6,5.25,2.3333333333333335,0.41935483870967744,5.792,click
|
| 26 |
+
docs/conf.py,0.82,59,50,0.05084745762711865,0.1016949152542373,0.0,0,100.0,0.0,0,0.0,0,0,2,6.75,2.3333333333333335,1.4408602150537635,6.56,click
|
| 27 |
+
tests/test_context.py,0.8,782,557,0.05370843989769821,0.2289002557544757,3.5833333333333335,9,26.13343350156297,8265.68566073093,24,6.625,4,1,12,7.75,4.333333333333333,3.5698924731182795,9.432,click
|
| 28 |
+
examples/termui/termui.py,0.78,169,129,0.005917159763313609,0.17159763313609466,2.7,8,42.253629046745274,855.5566984373538,10,12.6,5,0,4,4.0,0.6666666666666666,21.462365591397848,2.744,click
|
| 29 |
+
tests/test_chain.py,0.77,245,180,0.0,0.23673469387755103,2.7,3,42.0840388193055,313.33977418572846,10,1.3,4,1,3,5.0,2.0,5.376344086021505,5.352,click
|
| 30 |
+
examples/imagepipe/imagepipe.py,0.74,288,199,0.10416666666666667,0.17708333333333334,2.888888888888889,5,48.95065720286186,1761.5766022803014,18,9.777777777777779,4,0,5,2.25,0.6666666666666666,21.580645161290324,4.0,click
|
| 31 |
+
examples/repo/repo.py,0.72,166,113,0.1686746987951807,0.15060240963855423,2.1,5,67.33204405466785,193.39108448262166,10,9.9,3,0,4,3.75,0.6666666666666666,1.8387096774193548,3.088,click
|
| 32 |
+
tests/test_info_dict.py,0.71,345,319,0.002898550724637681,0.06376811594202898,2.5,5,46.01173723486477,179.29323028347633,8,1.0,2,1,2,2.0,2.3333333333333335,0.3225806451612903,3.016,click
|
| 33 |
+
examples/aliases/aliases.py,0.7,143,86,0.11888111888111888,0.22377622377622378,1.875,7,66.73409260824235,133.25742227201613,16,9.0625,4,0,3,4.0,1.6666666666666667,1.8387096774193548,2.192,click
|
| 34 |
+
tests/test_imports.py,0.69,80,70,0.0,0.125,5.0,5,62.329586036881174,48.6,1,11.0,3,1,4,7.75,4.333333333333333,1.8387096774193548,1.88,click
|
| 35 |
+
examples/complex/complex/cli.py,0.62,60,43,0.0,0.23333333333333334,2.5,4,51.55307621578065,76.10749561002055,8,7.5,4,0,3,2.75,1.0,12.344086021505376,1.04,click
|
| 36 |
+
src/click/_textwrap.py,0.61,188,131,0.0,0.18085106382978725,8.833333333333334,31,50.21074725034268,11637.307462280256,6,52.166666666666664,8,0,6,1.75,1.0,1.6236559139784945,1.696,click
|
| 37 |
+
examples/validation/validation.py,0.56,48,34,0.0,0.1875,3.25,4,74.46900661627932,563.1625072679101,4,10.5,4,0,2,3.0,1.3333333333333333,1.7849462365591398,0.96,click
|
| 38 |
+
tests/conftest.py,0.51,8,5,0.0,0.375,1.0,1,100.0,0.0,1,2.0,1,0,2,2.75,1.3333333333333333,12.827956989247312,3.536,click
|
| 39 |
+
src/click/globals.py,0.47,67,28,0.0,0.3283582089552239,1.6666666666666667,3,84.63937839905921,54.0,6,7.333333333333333,3,0,4,2.75,1.0,10.010752688172044,1.0,click
|
| 40 |
+
tests/test_normalization.py,0.45,63,48,0.015873015873015872,0.2222222222222222,4.0,8,59.129075445325064,195.34452978042594,3,1.0,2,1,1,1.5,1.3333333333333333,6.838709677419355,0.856,click
|
| 41 |
+
tests/test_deprecations.py,0.38,255,213,0.043137254901960784,0.13333333333333333,1.6153846153846154,3,56.198721731289254,51.2,13,2.923076923076923,3,1,8,1.25,0.6666666666666666,0.1827956989247312,2.28,click
|
| 42 |
+
examples/colors/colors.py,0.35,39,29,0.0,0.15384615384615385,4.0,4,100.0,0.0,1,15.0,2,0,1,1.75,1.0,22.182795698924732,0.808,click
|
| 43 |
+
tests/test_stream_lifecycle.py,0.33,540,323,0.11666666666666667,0.24444444444444444,3.5185185185185186,7,36.49326840190735,4568.884549703529,27,5.111111111111111,3,1,13,0.5,0.3333333333333333,1.4623655913978495,4.336,click
|
| 44 |
+
tests/test_utils/test_echo.py,0.33,172,125,0.046511627906976744,0.21511627906976744,7.0,25,44.311102246774745,574.3048115887916,6,5.333333333333333,2,1,7,0.5,0.6666666666666666,1.010752688172043,1.552,click
|
| 45 |
+
examples/naval/naval.py,0.32,72,43,0.0,0.2361111111111111,1.0,1,100.0,0.0,8,2.625,1,0,1,1.5,0.6666666666666666,25.419354838709676,1.216,click
|
| 46 |
+
examples/inout/inout.py,0.3,30,12,0.0,0.16666666666666666,4.0,4,97.41779417468767,1.0,1,24.0,4,0,1,1.25,1.3333333333333333,25.903225806451612,0.416,click
|
| 47 |
+
tests/test_abort_interrupt.py,0.29,179,86,0.0,0.29608938547486036,1.8461538461538463,5,68.53298782365535,321.0894851215667,13,8.538461538461538,3,1,3,0.25,0.3333333333333333,0.43010752688172044,1.432,click
|
| 48 |
+
tests/test_utils/test_echo_via_pager.py,0.28,259,211,0.09266409266409266,0.138996138996139,2.0,5,60.69927824428672,152.92539048396907,8,6.0,2,1,10,0.5,0.6666666666666666,0.5806451612903226,2.12,click
|
| 49 |
+
tests/test_types/test_Path.py,0.28,190,138,0.0,0.22631578947368422,2.2222222222222223,3,52.18810870570842,595.1026719031277,9,9.777777777777779,2,1,9,0.5,0.6666666666666666,0.34408602150537637,1.52,click
|
| 50 |
+
src/click/_utils.py,0.26,45,18,0.0,0.35555555555555557,1.2,2,100.0,0.0,5,6.0,2,0,3,1.0,0.6666666666666666,0.45161290322580644,0.44,click
|
| 51 |
+
tests/test_utils/test_open_file.py,0.25,117,83,0.017094017094017096,0.27350427350427353,2.25,5,48.81880582209984,497.809234850837,8,2.75,2,1,6,0.5,0.6666666666666666,0.8817204301075269,1.848,click
|
| 52 |
+
examples/complex/complex/commands/cmd_status.py,0.25,11,7,0.0,0.2727272727272727,1.0,1,100.0,0.0,1,4.0,1,0,2,1.25,1.3333333333333333,25.903225806451612,0.2,click
|
| 53 |
+
tests/typing/typing_path.py,0.23,34,23,0.08823529411764706,0.23529411764705882,0.0,0,78.17971375002185,42.793987519471216,0,0.0,0,0,4,0.25,0.3333333333333333,0.34408602150537637,0.272,click
|
| 54 |
+
tests/test_utils/test_style.py,0.22,127,120,0.05511811023622047,0.05511811023622047,2.0,3,77.40470878002992,12.63309714925922,3,1.6666666666666667,1,1,2,0.75,0.6666666666666666,1.0,1.032,click
|
| 55 |
+
tests/test_types/test_convert_type.py,0.21,108,71,0.23148148148148148,0.17592592592592593,2.4,3,82.73801938511949,39.863137138648355,5,2.6,1,1,4,0.25,0.3333333333333333,0.3763440860215054,0.864,click
|
| 56 |
+
tests/test_types/test_File.py,0.19,31,21,0.25806451612903225,0.16129032258064516,1.0,1,100.0,2.3774437510817346,1,19.0,1,1,2,0.25,0.3333333333333333,0.3763440860215054,0.248,click
|
| 57 |
+
examples/completion/completion.py,0.19,56,32,0.08928571428571429,0.3392857142857143,1.7142857142857142,4,76.65469465101006,43.47325716263743,7,3.857142857142857,3,0,3,0.75,0.3333333333333333,21.096774193548388,0.512,click
|
| 58 |
+
tests/test_command_decorators.py,0.17,92,68,0.0,0.2608695652173913,3.4,5,44.1786456575496,192.64744592513122,5,1.0,2,1,2,1.0,1.0,12.204301075268818,0.736,click
|
| 59 |
+
tests/test_utils/test_sentinel.py,0.15,97,75,0.07216494845360824,0.14432989690721648,5.666666666666667,11,65.77598084172764,655.3945306455817,3,1.0,2,1,8,0.5,0.6666666666666666,0.45161290322580644,0.776,click
|
| 60 |
+
tests/test_types/test_FloatRange.py,0.14,42,32,0.0,0.23809523809523808,1.6666666666666667,2,64.41888735626253,15.509775004326936,3,3.6666666666666665,1,1,2,0.25,0.3333333333333333,0.3763440860215054,0.336,click
|
| 61 |
+
tests/test_utils/test_KeepOpenFile.py,0.13,10,8,0.0,0.2,3.0,3,75.15517382051522,2.3774437510817346,1,6.0,2,1,1,0.5,0.6666666666666666,0.9247311827956989,0.096,click
|
| 62 |
+
tests/typing/typing_prompt.py,0.13,50,24,0.16,0.36,1.25,2,92.91946736989439,2.3774437510817346,4,3.75,2,0,2,0.25,0.3333333333333333,0.8602150537634409,0.4,click
|
| 63 |
+
tests/test_types/test_Choice.py,0.12,7,5,0.0,0.2857142857142857,2.0,2,79.7423427057672,2.3774437510817346,1,1.0,1,1,1,0.25,0.3333333333333333,0.3763440860215054,0.056,click
|
| 64 |
+
tests/test_types/test_ParamType.py,0.12,24,13,0.0,0.2916666666666667,3.0,3,88.79358865017207,83.25989365817458,2,1.0,3,1,2,0.25,0.3333333333333333,0.3763440860215054,0.192,click
|
| 65 |
+
tests/test_exceptions/test_FileError.py,0.12,6,4,0.0,0.3333333333333333,2.0,2,81.85633424453339,2.3774437510817346,1,1.0,1,1,1,0.25,0.3333333333333333,0.3763440860215054,0.048,click
|
| 66 |
+
tests/test_custom_classes.py,0.12,112,63,0.0,0.2767857142857143,2.3333333333333335,3,70.49873164207317,195.40466561840492,6,1.0,3,1,1,0.25,0.3333333333333333,24.236559139784948,0.896,click
|
| 67 |
+
examples/complex/complex/commands/cmd_init.py,0.11,13,9,0.0,0.23076923076923078,2.0,2,73.1756852057256,2.3774437510817346,1,5.0,2,0,2,1.25,1.0,25.419354838709676,0.232,click
|
| 68 |
+
tests/test_types/test_FuncParamType.py,0.11,22,16,0.0,0.2727272727272727,2.0,2,73.1756852057256,2.3774437510817346,1,1.0,2,1,2,0.25,0.3333333333333333,0.3763440860215054,0.176,click
|
| 69 |
+
tests/test_utils/test_make_default_short_help.py,0.11,63,57,0.015873015873015872,0.07936507936507936,4.0,4,76.3330396377789,15.509775004326936,1,1.0,2,1,2,0.75,1.0,0.3118279569892473,0.552,click
|
| 70 |
+
tests/test_utils/test__detect_program_name.py,0.11,27,20,0.0,0.25925925925925924,1.6666666666666667,2,73.041182281749,2.3774437510817346,3,2.6666666666666665,2,1,3,0.25,0.3333333333333333,1.021505376344086,0.216,click
|
| 71 |
+
tests/test_types/test_IntRange.py,0.1,38,32,0.0,0.15789473684210525,2.0,2,69.4458918400094,15.509775004326936,2,1.0,1,1,2,0.25,0.3333333333333333,0.3763440860215054,0.304,click
|
| 72 |
+
tests/typing/typing_aliased_group.py,0.1,46,30,0.021739130434782608,0.30434782608695654,2.6666666666666665,6,66.7458196448301,54.81181356564148,6,7.0,4,0,3,0.75,1.0,6.043010752688172,0.384,click
|
| 73 |
+
tests/typing/typing_group_kw_options.py,0.1,11,6,0.0,0.45454545454545453,1.0,1,100.0,0.0,1,2.0,1,0,2,0.25,0.3333333333333333,12.688172043010752,0.088,click
|
| 74 |
+
tests/typing/typing_progressbar.py,0.1,24,16,0.0,0.3333333333333333,1.6666666666666667,2,100.0,0.0,3,4.0,2,1,4,0.25,0.3333333333333333,7.537634408602151,0.192,click
|
| 75 |
+
tests/test_utils/test_LazyFile.py,0.1,11,9,0.0,0.18181818181818182,3.0,3,74.03933453534945,2.3774437510817346,1,7.0,2,1,1,0.5,0.6666666666666666,0.9247311827956989,0.104,click
|
| 76 |
+
tests/test_utils/test_prompt.py,0.09,100,62,0.12,0.25,2.1666666666666665,3,70.71511903450501,68.53238859703687,6,1.0,3,1,5,0.5,0.6666666666666666,0.8602150537634409,0.8,click
|
| 77 |
+
tests/typing/typing_confirmation_option.py,0.08,14,7,0.0,0.42857142857142855,1.0,1,100.0,0.0,1,2.0,1,0,2,0.75,1.0,6.043010752688172,0.128,click
|
| 78 |
+
tests/typing/typing_version_option.py,0.08,16,7,0.0,0.375,1.0,1,100.0,0.0,1,2.0,1,0,2,0.75,1.0,6.043010752688172,0.144,click
|
| 79 |
+
tests/test_parser.py,0.08,32,25,0.0,0.21875,2.0,2,63.37475596230147,12.63309714925922,3,1.0,1,1,4,0.75,1.0,12.333333333333334,0.32,click
|
| 80 |
+
tests/typing/typing_password_option.py,0.06,14,8,0.0,0.42857142857142855,1.0,1,100.0,0.0,1,2.0,1,0,3,0.25,0.3333333333333333,12.741935483870968,0.112,click
|
| 81 |
+
tests/typing/typing_edit.py,0.06,10,6,0.1,0.3,0.0,0,100.0,0.0,0,0.0,0,0,3,0.25,0.3333333333333333,0.5591397849462365,0.08,click
|
| 82 |
+
tests/test_utils/test_confirm.py,0.06,49,38,0.0,0.22448979591836735,7.5,13,49.652669125649695,236.60752504759634,2,1.0,3,1,1,0.25,0.3333333333333333,1.021505376344086,0.392,click
|
| 83 |
+
tests/typing/typing_help_option.py,0.05,13,7,0.0,0.38461538461538464,1.0,1,100.0,0.0,1,3.0,1,0,2,0.25,0.3333333333333333,12.741935483870968,0.104,click
|
| 84 |
+
tests/typing/typing_simple_example.py,0.05,17,9,0.0,0.35294117647058826,2.0,2,100.0,0.0,1,4.0,2,0,2,0.5,0.6666666666666666,10.010752688172044,0.136,click
|
| 85 |
+
tests/typing/typing_options.py,0.05,16,9,0.0,0.375,2.0,2,100.0,0.0,1,3.0,2,0,2,0.75,1.0,6.043010752688172,0.144,click
|
| 86 |
+
tests/test_utils/test_format_filename.py,0.04,9,7,0.0,0.2222222222222222,6.0,6,69.01468614621594,29.718046888521677,1,1.0,1,1,1,0.25,0.3333333333333333,1.021505376344086,0.072,click
|
| 87 |
+
tests/test_utils/test__expand_args.py,0.04,17,13,0.058823529411764705,0.17647058823529413,7.0,7,79.3806344361897,68.53238859703687,1,1.0,1,1,2,0.25,0.3333333333333333,1.021505376344086,0.136,click
|
| 88 |
+
tests/test_types/__init__.py,0.02,0,0,0.0,0.0,0.0,0,100.0,0.0,0,0.0,0,0,0,0.25,0.3333333333333333,0.3763440860215054,0.0,click
|
| 89 |
+
tests/test_exceptions/__init__.py,0.02,0,0,0.0,0.0,0.0,0,100.0,0.0,0,0.0,0,0,0,0.25,0.3333333333333333,0.3763440860215054,0.0,click
|
| 90 |
+
tests/test_utils/__init__.py,0.0,0,0,0.0,0.0,0.0,0,100.0,0.0,0,0.0,0,0,0,0.25,0.3333333333333333,1.021505376344086,0.0,click
|
| 91 |
+
examples/complex/complex/__init__.py,0.0,0,0,0.0,0.0,0.0,0,100.0,0.0,0,0.0,0,0,0,0.25,0.3333333333333333,48.645161290322584,0.0,click
|
| 92 |
+
examples/complex/complex/commands/__init__.py,0.0,0,0,0.0,0.0,0.0,0,100.0,0.0,0,0.0,0,0,0,0.25,0.3333333333333333,48.645161290322584,0.0,click
|
examples/psf__requests.csv
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
file,danger,loc,sloc,comment_ratio,blank_ratio,cyclomatic_complexity,max_complexity,maintainability_index,halstead_effort,num_functions,avg_function_len,max_nesting_depth,is_tested,dependency_count,commit_count,author_count,days_since_change,churn_rate,repo
|
| 2 |
+
src/requests/models.py,0.91,1184,720,0.23057432432432431,0.16554054054054054,3.9285714285714284,21,16.424806214955883,23812.207112680357,56,35.05357142857143,8,0,28,3.1666666666666665,3.0,0.76875,8.75,requests
|
| 3 |
+
src/requests/utils.py,0.9,1155,637,0.24935064935064935,0.2077922077922078,4.413043478260869,19,22.5697327312295,35511.65780635283,46,20.67391304347826,6,0,31,3.1666666666666665,3.5,0.775,8.343023255813954,requests
|
| 4 |
+
tests/test_utils.py,0.89,1013,830,0.044422507403751234,0.1362290227048371,2.324675324675325,6,18.409681277306113,9958.25440156326,77,5.64935064935065,5,1,18,12.0,15.5,0.95,14.290697674418604,requests
|
| 5 |
+
tests/test_requests.py,0.87,3094,2440,0.048157724628312866,0.16063348416289594,2.859437751004016,8,0.0,109553.96156774365,249,14.012048192771084,7,1,32,45.833333333333336,47.0,0.76875,58.74418604651163,requests
|
| 6 |
+
src/requests/adapters.py,0.87,748,424,0.28475935828877,0.15240641711229946,3.8636363636363638,20,46.100395693040014,1842.6083666723753,22,55.54545454545455,6,0,28,3.0,3.5,0.775,6.034883720930233,requests
|
| 7 |
+
docs/conf.py,0.82,385,84,0.5584415584415584,0.24415584415584415,0.0,0,100.0,0.0,0,0.0,0,0,3,6.666666666666667,7.5,1.1375,4.738372093023256,requests
|
| 8 |
+
setup.py,0.79,9,6,0.1111111111111111,0.3333333333333333,0.0,0,99.90348152172656,2.3774437510817346,0,0.0,1,0,2,29.0,21.5,1.5375,6.773255813953488,requests
|
| 9 |
+
tests/test_lowlevel.py,0.79,428,284,0.10514018691588785,0.23130841121495327,3.4285714285714284,6,44.153518212593404,2802.577552678092,14,21.928571428571427,2,1,6,4.5,6.0,1.58125,4.494186046511628,requests
|
| 10 |
+
tests/testserver/server.py,0.73,176,137,0.045454545454545456,0.18181818181818182,2.2,4,52.34542381192951,190.66666666666666,15,19.8,4,0,4,2.5,4.0,5.875,1.7674418604651163,requests
|
| 11 |
+
src/requests/sessions.py,0.7,920,479,0.30869565217391304,0.17608695652173914,4.225806451612903,15,35.48718644321636,10488.742764470222,31,49.61290322580645,5,0,24,1.8333333333333333,3.0,0.775,6.372093023255814,requests
|
| 12 |
+
tests/test_testserver.py,0.69,165,112,0.04242424242424243,0.21818181818181817,2.4166666666666665,4,49.208813990549096,111.4694169837047,12,18.333333333333332,3,1,6,2.8333333333333335,3.0,1.58125,2.0174418604651163,requests
|
| 13 |
+
src/requests/__init__.py,0.54,219,137,0.2237442922374429,0.1598173515981735,6.5,10,69.51805483330125,455.0171144033819,2,23.0,3,0,19,1.5,2.5,0.775,1.680232558139535,requests
|
| 14 |
+
src/requests/_types.py,0.51,188,126,0.11702127659574468,0.2127659574468085,1.4166666666666667,3,52.3424609035083,5317.0824764809,12,4.5,2,0,10,1.1666666666666667,1.0,0.76875,1.2209302325581395,requests
|
| 15 |
+
tests/test_structures.py,0.51,91,69,0.0,0.21978021978021978,2.1176470588235294,3,41.14719594099197,520.9561949440348,17,6.235294117647059,2,1,2,1.1666666666666667,2.0,1.01875,1.680232558139535,requests
|
| 16 |
+
src/requests/__version__.py,0.5,14,10,0.21428571428571427,0.07142857142857142,0.0,0,100.0,0.0,0,0.0,0,0,0,2.3333333333333335,1.0,0.93125,0.36046511627906974,requests
|
| 17 |
+
tests/conftest.py,0.48,67,43,0.05970149253731343,0.2835820895522388,1.2,2,72.68279430886432,15.509775004326936,5,8.2,2,0,8,1.0,2.0,0.95625,0.5174418604651163,requests
|
| 18 |
+
src/requests/status_codes.py,0.38,128,100,0.1640625,0.09375,6.0,6,83.6791512823983,112.92631877271643,1,16.0,4,0,1,0.6666666666666666,1.5,1.0,0.8255813953488372,requests
|
| 19 |
+
src/requests/auth.py,0.38,354,238,0.11864406779661017,0.20056497175141244,2.739130434782609,19,42.76044354466967,7330.84285060853,23,23.47826086956522,8,0,16,0.8333333333333334,1.0,0.775,2.604651162790698,requests
|
| 20 |
+
src/requests/packages.py,0.38,23,15,0.17391304347826086,0.17391304347826086,0.0,0,86.66611985445543,89.62406251802892,0,0.0,3,0,2,1.0,1.5,5.49375,0.4011627906976744,requests
|
| 21 |
+
src/requests/exceptions.py,0.36,162,45,0.18518518518518517,0.3950617283950617,1.2857142857142858,5,73.32411200242946,196.0,28,3.9285714285714284,3,0,5,0.6666666666666666,1.0,1.0,1.0,requests
|
| 22 |
+
src/requests/cookies.py,0.32,625,343,0.2624,0.1952,2.3214285714285716,9,38.08781688720079,9077.945691552755,56,15.375,7,0,13,0.5,1.0,0.775,4.563953488372093,requests
|
| 23 |
+
src/requests/compat.py,0.3,115,66,0.2608695652173913,0.1826086956521739,4.0,4,79.26713569292579,39.863137138648355,1,10.0,4,0,18,1.1666666666666667,1.0,0.775,0.8546511627906976,requests
|
| 24 |
+
src/requests/api.py,0.29,180,38,0.5611111111111111,0.22777777777777777,1.0,1,100.0,0.0,8,17.875,1,0,6,0.3333333333333333,0.5,1.0,1.1395348837209303,requests
|
| 25 |
+
tests/__init__.py,0.25,14,6,0.2857142857142857,0.21428571428571427,0.0,0,100.0,0.0,0,0.0,1,0,2,1.1666666666666667,2.5,8.4875,0.3372093023255814,requests
|
| 26 |
+
src/requests/help.py,0.23,132,99,0.11363636363636363,0.16666666666666666,4.333333333333333,6,61.949053948140005,550.713459778579,3,29.333333333333332,5,0,13,0.8333333333333334,1.0,0.775,1.0348837209302326,requests
|
| 27 |
+
tests/compat.py,0.2,23,19,0.0,0.17391304347826086,1.0,1,100.0,0.0,1,10.0,1,0,4,0.6666666666666666,1.0,10.15625,0.2616279069767442,requests
|
| 28 |
+
src/requests/_internal_utils.py,0.17,51,25,0.0,0.21568627450980393,2.5,3,100.0,0.0,2,12.0,2,0,2,0.6666666666666666,0.5,1.0,0.36627906976744184,requests
|
| 29 |
+
src/requests/structures.py,0.16,130,62,0.26153846153846155,0.26153846153846155,1.368421052631579,2,76.39007946622573,42.793987519471216,19,8.68421052631579,3,0,5,0.3333333333333333,0.5,1.0,0.9883720930232558,requests
|
| 30 |
+
tests/test_adapters.py,0.16,8,5,0.0,0.25,2.0,2,78.01508585192973,2.3774437510817346,1,1.0,1,1,1,0.3333333333333333,1.0,0.95625,0.06976744186046512,requests
|
| 31 |
+
src/requests/certs.py,0.15,18,3,0.5555555555555556,0.2777777777777778,0.0,0,64.3595984114544,2.3774437510817346,0,0.0,1,0,1,0.3333333333333333,0.5,1.58125,0.10465116279069768,requests
|
| 32 |
+
docs/_themes/flask_theme_support.py,0.12,86,73,0.7674418604651163,0.13953488372093023,1.0,1,100.0,0.0,1,79.0,1,0,2,0.16666666666666666,0.5,35.16875,0.5,requests
|
| 33 |
+
tests/utils.py,0.1,17,15,0.0,0.11764705882352941,3.0,3,69.19993388914534,2.3774437510817346,1,12.0,3,0,2,0.5,1.0,10.15625,0.12209302325581395,requests
|
| 34 |
+
tests/test_help.py,0.09,27,13,0.0,0.3333333333333333,1.8,2,86.45398139061467,27.0,5,2.8,2,1,2,0.8333333333333334,1.0,7.2125,0.47093023255813954,requests
|
| 35 |
+
src/requests/hooks.py,0.09,48,25,0.1875,0.2708333333333333,4.0,6,84.90733657858264,68.33680652339717,2,9.5,4,0,5,0.5,0.5,1.0,0.38372093023255816,requests
|
| 36 |
+
tests/test_packages.py,0.06,13,7,0.0,0.46153846153846156,1.0,1,100.0,0.0,3,2.0,1,1,1,0.16666666666666666,0.5,21.38125,0.0755813953488372,requests
|
| 37 |
+
tests/test_hooks.py,0.05,22,15,0.0,0.3181818181818182,1.6666666666666667,2,67.28666784000676,6.965784284662087,3,1.3333333333333333,1,1,2,0.5,1.0,10.15625,0.23255813953488372,requests
|
| 38 |
+
tests/testserver/__init__.py,0.0,0,0,0.0,0.0,0.0,0,100.0,0.0,0,0.0,0,0,0,0.16666666666666666,0.5,24.20625,0.0,requests
|
fast_git.py
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
fast_git.py - same output as feature_lib.feature_git_raw, but fast on big repos.
|
| 3 |
+
|
| 4 |
+
feature_git_raw calls commit.stats (one `git diff` per commit) for every commit of
|
| 5 |
+
every file, so a commit touching 50 files is diffed 50 times. On a repo with
|
| 6 |
+
thousands of commits that takes many minutes. Here:
|
| 7 |
+
|
| 8 |
+
* churn: ONE `git log --numstat` over the whole history, diffing each commit
|
| 9 |
+
against its first parent (or the empty tree for root commits) with renames
|
| 10 |
+
off, which is exactly what GitPython's commit.stats does.
|
| 11 |
+
* per-file history: one `git log -- <file>` per file, which applies the same
|
| 12 |
+
history simplification as repo.iter_commits(paths=file).
|
| 13 |
+
|
| 14 |
+
Raw author names (%an, no mailmap) and the committer-local date (%cI) match
|
| 15 |
+
commit.author.name and commit.committed_datetime.
|
| 16 |
+
"""
|
| 17 |
+
|
| 18 |
+
import subprocess
|
| 19 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 20 |
+
from datetime import date
|
| 21 |
+
from pathlib import Path
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def _git(repo_path: Path, *args: str) -> str:
|
| 25 |
+
return subprocess.run(
|
| 26 |
+
["git", "-C", str(repo_path), *args],
|
| 27 |
+
check=True, capture_output=True, text=True, encoding="utf-8", errors="replace",
|
| 28 |
+
).stdout
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def churn_by_commit(repo_path: Path) -> dict[str, dict[str, int]]:
|
| 32 |
+
out = _git(
|
| 33 |
+
repo_path, "log", "HEAD", "--no-renames", "--diff-merges=first-parent",
|
| 34 |
+
"--numstat", "--format=%x00%H",
|
| 35 |
+
)
|
| 36 |
+
churn = {}
|
| 37 |
+
for block in out.split("\x00")[1:]:
|
| 38 |
+
lines = block.strip("\n").split("\n")
|
| 39 |
+
files = {}
|
| 40 |
+
for line in lines[1:]:
|
| 41 |
+
parts = line.split("\t", 2)
|
| 42 |
+
if len(parts) != 3:
|
| 43 |
+
continue
|
| 44 |
+
insertions, deletions, path = parts
|
| 45 |
+
# GitPython counts binary files ("-") as 0
|
| 46 |
+
files[path] = (0 if insertions == "-" else int(insertions)) + (0 if deletions == "-" else int(deletions))
|
| 47 |
+
churn[lines[0]] = files
|
| 48 |
+
return churn
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def file_history(repo_path: Path, file: str) -> list[tuple[str, str, str]]:
|
| 52 |
+
out = _git(repo_path, "log", "HEAD", "--format=%H%x09%cI%x09%an", "--", file)
|
| 53 |
+
return [tuple(line.split("\t", 2)) for line in out.splitlines() if line]
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def git_features(repo_path: Path, files: list[str], workers: int = 4) -> dict[str, dict]:
|
| 57 |
+
churn = churn_by_commit(repo_path)
|
| 58 |
+
posix_files = [f.replace("\\", "/") for f in files]
|
| 59 |
+
|
| 60 |
+
with ThreadPoolExecutor(max_workers=workers) as pool:
|
| 61 |
+
histories = list(pool.map(lambda f: file_history(repo_path, f), posix_files))
|
| 62 |
+
|
| 63 |
+
features = {}
|
| 64 |
+
for file, posix, history in zip(files, posix_files, histories):
|
| 65 |
+
authors = set()
|
| 66 |
+
churn_total = 0
|
| 67 |
+
for sha, _, author in history:
|
| 68 |
+
authors.add(author)
|
| 69 |
+
churn_total += churn.get(sha, {}).get(posix, 0)
|
| 70 |
+
|
| 71 |
+
features[file] = {
|
| 72 |
+
"commit_count": len(history),
|
| 73 |
+
"author_count": len(authors),
|
| 74 |
+
"days_since_change": (date.today() - date.fromisoformat(history[0][1][:10])).days if history else 0,
|
| 75 |
+
"churn_rate": churn_total,
|
| 76 |
+
}
|
| 77 |
+
return features
|
metrics_worker.py
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
metrics_worker.py - computes the per-file code features in a separate process.
|
| 3 |
+
|
| 4 |
+
Radon can hang on some files (e.g. a 7k-line test file in fastapi never finishes
|
| 5 |
+
in radon.raw.analyze), and a stuck thread can't be killed. app.py runs this
|
| 6 |
+
worker as a subprocess so it can kill it when a file takes too long.
|
| 7 |
+
|
| 8 |
+
Protocol: argv[1] = repo path, stdin = one relative file path per line,
|
| 9 |
+
stdout = one JSON object per file, in the same order.
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
import json
|
| 13 |
+
import sys
|
| 14 |
+
from pathlib import Path
|
| 15 |
+
|
| 16 |
+
HERE = Path(__file__).resolve().parent
|
| 17 |
+
for candidate in (HERE / "src", HERE.parent / "src"):
|
| 18 |
+
if candidate.is_dir():
|
| 19 |
+
sys.path.insert(0, str(candidate))
|
| 20 |
+
break
|
| 21 |
+
|
| 22 |
+
from feature_lib import embedding_extract_functions, features_ats, features_radon
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def main():
|
| 26 |
+
repo_path = Path(sys.argv[1])
|
| 27 |
+
for file in sys.stdin.read().splitlines():
|
| 28 |
+
try:
|
| 29 |
+
code = (repo_path / file).read_text(encoding="utf-8")
|
| 30 |
+
features = {}
|
| 31 |
+
features.update(features_radon(code))
|
| 32 |
+
features.update(features_ats(code))
|
| 33 |
+
out = {"file": file, "features": features, "functions": embedding_extract_functions(code)}
|
| 34 |
+
except Exception as e:
|
| 35 |
+
out = {"file": file, "error": f"{type(e).__name__}: {e}"}
|
| 36 |
+
print(json.dumps(out, default=float), flush=True)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
if __name__ == "__main__":
|
| 40 |
+
main()
|
models/features_v3.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0045765fd8165c4c9ceaf89f323d64745e2bd611c048ff3a71a0d3ece72e830d
|
| 3 |
+
size 291
|
models/pca_v3.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e29e6ee614256d9ac9bb3efcf53188224b6a93b86d088022222f9b945676cccc
|
| 3 |
+
size 65918
|
models/random_forest_v3.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f2fe5bd76d6ef4d75398b6721e03a0f8d7f601ce7e5894aa82503a8751809f4e
|
| 3 |
+
size 2996649
|
requirements.txt
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ZeroGPU supports torch 2.8 - 2.13 (the `spaces` package is preinstalled there)
|
| 2 |
+
torch==2.13.0
|
| 3 |
+
# Must match the version the v3 .pkl artifacts were trained with
|
| 4 |
+
scikit-learn==1.9.0
|
| 5 |
+
sentence-transformers
|
| 6 |
+
pandas
|
| 7 |
+
numpy
|
| 8 |
+
gitpython
|
| 9 |
+
radon
|
| 10 |
+
joblib
|
src/feature_lib.py
ADDED
|
@@ -0,0 +1,190 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ast
|
| 2 |
+
from radon.complexity import cc_visit
|
| 3 |
+
from radon import metrics, raw
|
| 4 |
+
from datetime import date
|
| 5 |
+
import numpy as np
|
| 6 |
+
|
| 7 |
+
class MyAnalyzer(ast.NodeVisitor):
|
| 8 |
+
def __init__(self):
|
| 9 |
+
self.dependency_counts = 0
|
| 10 |
+
self.max_nesting_depth = 0
|
| 11 |
+
self.current_depth = 0
|
| 12 |
+
self.is_tested = 0
|
| 13 |
+
|
| 14 |
+
def visit_Import(self, node):
|
| 15 |
+
self.dependency_counts += 1
|
| 16 |
+
self.generic_visit(node)
|
| 17 |
+
def visit_ImportFrom(self, node):
|
| 18 |
+
self.dependency_counts += 1
|
| 19 |
+
self.generic_visit(node)
|
| 20 |
+
|
| 21 |
+
def visit_If(self, node):
|
| 22 |
+
self.current_depth += 1
|
| 23 |
+
|
| 24 |
+
if self.current_depth > self.max_nesting_depth:
|
| 25 |
+
self.max_nesting_depth = self.current_depth
|
| 26 |
+
self.generic_visit(node)
|
| 27 |
+
self.current_depth -= 1
|
| 28 |
+
|
| 29 |
+
def visit_For(self, node):
|
| 30 |
+
self.current_depth += 1
|
| 31 |
+
|
| 32 |
+
if self.current_depth > self.max_nesting_depth:
|
| 33 |
+
self.max_nesting_depth = self.current_depth
|
| 34 |
+
self.generic_visit(node)
|
| 35 |
+
self.current_depth -= 1
|
| 36 |
+
def visit_While(self, node):
|
| 37 |
+
self.current_depth += 1
|
| 38 |
+
|
| 39 |
+
if self.current_depth > self.max_nesting_depth:
|
| 40 |
+
self.max_nesting_depth = self.current_depth
|
| 41 |
+
self.generic_visit(node)
|
| 42 |
+
self.current_depth -= 1
|
| 43 |
+
def visit_Try(self, node):
|
| 44 |
+
self.current_depth += 1
|
| 45 |
+
|
| 46 |
+
if self.current_depth > self.max_nesting_depth:
|
| 47 |
+
self.max_nesting_depth = self.current_depth
|
| 48 |
+
self.generic_visit(node)
|
| 49 |
+
self.current_depth -= 1
|
| 50 |
+
|
| 51 |
+
def visit_FunctionDef(self, node):
|
| 52 |
+
self.current_depth += 1
|
| 53 |
+
if self.current_depth > self.max_nesting_depth:
|
| 54 |
+
self.max_nesting_depth = self.current_depth
|
| 55 |
+
|
| 56 |
+
if node.name.startswith("test_"):
|
| 57 |
+
self.is_tested = 1
|
| 58 |
+
self.generic_visit(node)
|
| 59 |
+
self.current_depth -= 1
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def visit_ClassDef(self, node):
|
| 63 |
+
self.current_depth += 1
|
| 64 |
+
|
| 65 |
+
if self.current_depth > self.max_nesting_depth:
|
| 66 |
+
self.max_nesting_depth = self.current_depth
|
| 67 |
+
if node.name.startswith("Test"):
|
| 68 |
+
self.is_tested = 1
|
| 69 |
+
self.generic_visit(node)
|
| 70 |
+
self.current_depth -= 1
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def features_radon(file):
|
| 74 |
+
result_raw_loc = raw.analyze(file).loc # feat
|
| 75 |
+
result_raw_sloc = raw.analyze(file).sloc # feat
|
| 76 |
+
result_raw_comments = raw.analyze(file).comments
|
| 77 |
+
result_raw_multi = raw.analyze(file).multi
|
| 78 |
+
result_raw_blank = raw.analyze(file).blank
|
| 79 |
+
|
| 80 |
+
comment_ratio = (
|
| 81 |
+
(result_raw_comments + result_raw_multi) / result_raw_loc
|
| 82 |
+
if result_raw_comments > 0 else 0
|
| 83 |
+
)
|
| 84 |
+
blank_ratio = result_raw_blank / result_raw_loc if result_raw_loc > 0 else 0
|
| 85 |
+
# Cyclomatic complexity
|
| 86 |
+
result_cc = cc_visit(file)
|
| 87 |
+
if len(result_cc) > 0:
|
| 88 |
+
|
| 89 |
+
sum_func_lines = 0
|
| 90 |
+
complexities = [score.complexity for score in result_cc]
|
| 91 |
+
max_complexity = max(complexities) # feat
|
| 92 |
+
average_cc = sum(complexities) / len(complexities) # feat
|
| 93 |
+
for score in result_cc:
|
| 94 |
+
func_lines = score.endline - score.lineno + 1
|
| 95 |
+
sum_func_lines += func_lines
|
| 96 |
+
|
| 97 |
+
num_functions = len(result_cc) # feat
|
| 98 |
+
|
| 99 |
+
avg_function_len = sum_func_lines / num_functions # feat
|
| 100 |
+
else:
|
| 101 |
+
num_functions = 0
|
| 102 |
+
avg_function_len = 0
|
| 103 |
+
average_cc = 0
|
| 104 |
+
max_complexity = 0
|
| 105 |
+
|
| 106 |
+
# Maintainability Index
|
| 107 |
+
result_mi = metrics.mi_visit(file, multi=True) # feat
|
| 108 |
+
|
| 109 |
+
# Halstead metrics (effort specifically)
|
| 110 |
+
result_hl = metrics.h_visit(file).total.effort # feat
|
| 111 |
+
|
| 112 |
+
return {
|
| 113 |
+
"loc": result_raw_loc,
|
| 114 |
+
"sloc": result_raw_sloc,
|
| 115 |
+
"comment_ratio": comment_ratio,
|
| 116 |
+
"blank_ratio": blank_ratio,
|
| 117 |
+
"cyclomatic_complexity": average_cc,
|
| 118 |
+
"max_complexity": max_complexity,
|
| 119 |
+
"maintainability_index": result_mi,
|
| 120 |
+
"halstead_effort": result_hl,
|
| 121 |
+
"num_functions": num_functions,
|
| 122 |
+
"avg_function_len": avg_function_len,
|
| 123 |
+
}
|
| 124 |
+
|
| 125 |
+
def features_ats(file):
|
| 126 |
+
# ast dependency counts and max nesting depth
|
| 127 |
+
result_ast = ast.parse(file)
|
| 128 |
+
analyze = MyAnalyzer()
|
| 129 |
+
analyze.visit(result_ast)
|
| 130 |
+
max_nesting_depth = analyze.max_nesting_depth # feat
|
| 131 |
+
dependency_counts = analyze.dependency_counts #feat
|
| 132 |
+
nesting_depth = analyze.current_depth
|
| 133 |
+
is_tested = analyze.is_tested # feat
|
| 134 |
+
|
| 135 |
+
return {
|
| 136 |
+
"max_nesting_depth": max_nesting_depth,
|
| 137 |
+
"is_tested": is_tested,
|
| 138 |
+
"dependency_count": dependency_counts
|
| 139 |
+
}
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def feature_git_raw(file, repo):
|
| 144 |
+
author_list = set()
|
| 145 |
+
churn_rate_total = 0
|
| 146 |
+
|
| 147 |
+
file_commits = list(repo.iter_commits(paths=file))
|
| 148 |
+
commit_count = len(file_commits)
|
| 149 |
+
|
| 150 |
+
for commit in file_commits:
|
| 151 |
+
author_list.add(commit.author.name)
|
| 152 |
+
churn_rate = commit.stats.files
|
| 153 |
+
change_file = file.replace("\\", "/")
|
| 154 |
+
if change_file in churn_rate:
|
| 155 |
+
churn_rate_total += (
|
| 156 |
+
churn_rate[change_file]['deletions']
|
| 157 |
+
+ churn_rate[change_file]['insertions']
|
| 158 |
+
)
|
| 159 |
+
|
| 160 |
+
author_count = len(author_list)
|
| 161 |
+
|
| 162 |
+
if commit_count > 0:
|
| 163 |
+
days_since_change = (date.today() - file_commits[0].committed_datetime.date()).days
|
| 164 |
+
else:
|
| 165 |
+
days_since_change = 0
|
| 166 |
+
|
| 167 |
+
return {
|
| 168 |
+
"commit_count": commit_count,
|
| 169 |
+
"author_count": author_count,
|
| 170 |
+
"days_since_change": days_since_change,
|
| 171 |
+
"churn_rate": churn_rate_total,
|
| 172 |
+
}
|
| 173 |
+
|
| 174 |
+
def normalization_relative_to_repo(df_features, repo_col="repo"):
|
| 175 |
+
relative_cols = ["commit_count", "author_count", "churn_rate", "days_since_change"]
|
| 176 |
+
for col in relative_cols:
|
| 177 |
+
medians = df_features.groupby(repo_col)[col].transform("median")
|
| 178 |
+
medians = medians.replace(0, 1)
|
| 179 |
+
df_features[col] = df_features[col] / medians
|
| 180 |
+
return df_features
|
| 181 |
+
|
| 182 |
+
def embedding_extract_functions(code, min_lines=2):
|
| 183 |
+
functions = cc_visit(code)
|
| 184 |
+
lines = code.splitlines()
|
| 185 |
+
texts = []
|
| 186 |
+
for f in functions:
|
| 187 |
+
body = "\n".join(lines[f.lineno - 1 : f.endline])
|
| 188 |
+
if len(body.splitlines()) >= min_lines:
|
| 189 |
+
texts.append(body)
|
| 190 |
+
return texts
|
src/predict.py
ADDED
|
@@ -0,0 +1,281 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
predict.py - analyzes a local repo and predicts the technical debt per file
|
| 3 |
+
|
| 4 |
+
Uses model v3: tabular features (Radon + AST + relative git activity) combined
|
| 5 |
+
with PCA-reduced code embeddings (see docs/decisions/004-embeddings-tradeoff.md).
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
import argparse
|
| 9 |
+
import sys
|
| 10 |
+
import os
|
| 11 |
+
import shutil
|
| 12 |
+
import stat
|
| 13 |
+
import tempfile
|
| 14 |
+
import re
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
from datetime import datetime
|
| 17 |
+
|
| 18 |
+
import joblib
|
| 19 |
+
import numpy as np
|
| 20 |
+
import pandas as pd
|
| 21 |
+
from git import Repo
|
| 22 |
+
from git.exc import InvalidGitRepositoryError, NoSuchPathError
|
| 23 |
+
from sentence_transformers import SentenceTransformer
|
| 24 |
+
|
| 25 |
+
from feature_lib import (
|
| 26 |
+
features_ats,
|
| 27 |
+
features_radon,
|
| 28 |
+
feature_git_raw,
|
| 29 |
+
normalization_relative_to_repo,
|
| 30 |
+
embedding_extract_functions,
|
| 31 |
+
)
|
| 32 |
+
|
| 33 |
+
SRC_DIR = Path(__file__).resolve().parent
|
| 34 |
+
MODELS_DIR = SRC_DIR.parent / "models"
|
| 35 |
+
REPORTS_DIR = SRC_DIR.parent / "temp_reports"
|
| 36 |
+
|
| 37 |
+
if str(SRC_DIR) not in sys.path:
|
| 38 |
+
sys.path.insert(0, str(SRC_DIR))
|
| 39 |
+
|
| 40 |
+
# Oficial Artifacts: v3 (embeddings + PCA)
|
| 41 |
+
feat_list = joblib.load(MODELS_DIR / "features_v3.pkl")
|
| 42 |
+
MODEL = joblib.load(MODELS_DIR / "random_forest_v3.pkl")
|
| 43 |
+
PCA_MODEL = joblib.load(MODELS_DIR / "pca_v3.pkl")
|
| 44 |
+
EMBED_MODEL = SentenceTransformer("all-MiniLM-L6-v2")
|
| 45 |
+
|
| 46 |
+
CONFIG = {
|
| 47 |
+
"skip_dirs": ["test", ".venv"],
|
| 48 |
+
}
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def validate_repo(path: str) -> tuple[Path, Repo]:
|
| 52 |
+
repo_path = Path(path).resolve()
|
| 53 |
+
|
| 54 |
+
if not repo_path.exists():
|
| 55 |
+
print(f"Error: the path '{repo_path}' does not exist.")
|
| 56 |
+
sys.exit(1)
|
| 57 |
+
|
| 58 |
+
try:
|
| 59 |
+
repo_git = Repo(repo_path)
|
| 60 |
+
except (InvalidGitRepositoryError, NoSuchPathError):
|
| 61 |
+
print(f"Error: '{repo_path}' is not a valid git repository.")
|
| 62 |
+
sys.exit(1)
|
| 63 |
+
|
| 64 |
+
return repo_path, repo_git
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def find_python_files(repo_path: Path) -> list[str]:
|
| 68 |
+
return [
|
| 69 |
+
str(f.relative_to(repo_path)) for f in repo_path.rglob("*.py")
|
| 70 |
+
if not any(part in CONFIG["skip_dirs"] for part in f.parts)
|
| 71 |
+
]
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def print_summary(repo_path: Path, files: list[str]):
|
| 75 |
+
print("\n" + "=" * 60)
|
| 76 |
+
print(f" FILES FOUND IN {repo_path.name}")
|
| 77 |
+
print("=" * 60)
|
| 78 |
+
print(f" Total: {len(files)} .py files")
|
| 79 |
+
print("=" * 60)
|
| 80 |
+
|
| 81 |
+
for file in files[:15]:
|
| 82 |
+
print(f" {file}")
|
| 83 |
+
|
| 84 |
+
if len(files) > 15:
|
| 85 |
+
print(f" ... and {len(files) - 15} more")
|
| 86 |
+
|
| 87 |
+
print("=" * 60)
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def feature_extraction(path, files, repo_git, repo_name):
|
| 91 |
+
resultados = {}
|
| 92 |
+
todas_las_funciones = []
|
| 93 |
+
indice_por_archivo = []
|
| 94 |
+
|
| 95 |
+
for archivo in files:
|
| 96 |
+
try:
|
| 97 |
+
with open(path / archivo, "r", encoding="utf-8") as f:
|
| 98 |
+
feat_file = f.read()
|
| 99 |
+
|
| 100 |
+
features = {}
|
| 101 |
+
features.update(features_radon(feat_file))
|
| 102 |
+
features.update(features_ats(feat_file))
|
| 103 |
+
features.update(feature_git_raw(archivo, repo_git))
|
| 104 |
+
features["repo"] = repo_name
|
| 105 |
+
|
| 106 |
+
# Ruta normalizada — evita el desajuste backslash/forward-slash
|
| 107 |
+
# entre este dict y las claves de embeddings_por_archivo.
|
| 108 |
+
clave = f"{repo_name}/{archivo}".replace("\\", "/")
|
| 109 |
+
resultados[clave] = features
|
| 110 |
+
|
| 111 |
+
funciones_archivo = embedding_extract_functions(feat_file)
|
| 112 |
+
indice_por_archivo.append((clave, len(funciones_archivo)))
|
| 113 |
+
todas_las_funciones.extend(funciones_archivo)
|
| 114 |
+
|
| 115 |
+
except Exception as e:
|
| 116 |
+
print(f"corrupted file: {archivo} , {e}")
|
| 117 |
+
continue
|
| 118 |
+
|
| 119 |
+
return resultados, todas_las_funciones, indice_por_archivo
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def generar_embeddings_reducidos(todas_las_funciones, indice_por_archivo):
|
| 123 |
+
"""Genera embeddings por función (batched), hace mean pooling por archivo.
|
| 124 |
+
NO aplica PCA aquí — eso ocurre en full_report, sobre la matriz completa."""
|
| 125 |
+
if todas_las_funciones:
|
| 126 |
+
embeddings_funciones = EMBED_MODEL.encode(todas_las_funciones, batch_size=64)
|
| 127 |
+
else:
|
| 128 |
+
embeddings_funciones = np.empty((0, 384))
|
| 129 |
+
|
| 130 |
+
embeddings_por_archivo = {}
|
| 131 |
+
cursor = 0
|
| 132 |
+
for clave, cantidad in indice_por_archivo:
|
| 133 |
+
if cantidad == 0:
|
| 134 |
+
embeddings_por_archivo[clave] = np.zeros(384)
|
| 135 |
+
continue
|
| 136 |
+
bloque = embeddings_funciones[cursor: cursor + cantidad]
|
| 137 |
+
embeddings_por_archivo[clave] = bloque.mean(axis=0)
|
| 138 |
+
cursor += cantidad
|
| 139 |
+
|
| 140 |
+
return embeddings_por_archivo
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def full_report(results: dict, embeddings_por_archivo: dict) -> pd.DataFrame:
|
| 144 |
+
"""Combina features tabulares (normalizados relativo al repo) con embeddings
|
| 145 |
+
reducidos por PCA (ya ajustado en entrenamiento — aquí solo .transform())."""
|
| 146 |
+
df = pd.DataFrame.from_dict(results, orient="index")
|
| 147 |
+
df.index.name = "file"
|
| 148 |
+
|
| 149 |
+
df = normalization_relative_to_repo(df, repo_col="repo")
|
| 150 |
+
|
| 151 |
+
X_tabular = df[feat_list].values
|
| 152 |
+
matriz_embeddings = np.array([embeddings_por_archivo[clave] for clave in df.index])
|
| 153 |
+
X_embeddings_reducido = PCA_MODEL.transform(matriz_embeddings) # transform, NUNCA fit_transform
|
| 154 |
+
|
| 155 |
+
X_final = np.hstack([X_tabular, X_embeddings_reducido])
|
| 156 |
+
df.insert(0, "danger", MODEL.predict_proba(X_final)[:, 1].round(3))
|
| 157 |
+
|
| 158 |
+
return df.sort_values("danger", ascending=False).reset_index()
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
def save_csv_report(report: pd.DataFrame, repo_name: str, destination: Path = REPORTS_DIR) -> Path:
|
| 162 |
+
destination.mkdir(parents=True, exist_ok=True)
|
| 163 |
+
clean_name = re.sub(r"[^A-Za-z0-9._-]", "_", repo_name) or "repo"
|
| 164 |
+
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
| 165 |
+
csv_path = destination / f"report_{clean_name}_{timestamp}.csv"
|
| 166 |
+
report.to_csv(csv_path, index=False, encoding="utf-8")
|
| 167 |
+
return csv_path
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def _delete_tree(path: Path):
|
| 171 |
+
def _remove_readonly(func, path, _exc_info):
|
| 172 |
+
os.chmod(path, stat.S_IWRITE)
|
| 173 |
+
func(path)
|
| 174 |
+
|
| 175 |
+
shutil.rmtree(path, onerror=_remove_readonly)
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def fetch_and_clone_repo(max_attempts: int = 5):
|
| 179 |
+
try:
|
| 180 |
+
from repo_search import get_random_repo
|
| 181 |
+
except EnvironmentError as e:
|
| 182 |
+
print(f"Error: {e}")
|
| 183 |
+
return None
|
| 184 |
+
|
| 185 |
+
tried_urls = set()
|
| 186 |
+
|
| 187 |
+
for attempt in range(1, max_attempts + 1):
|
| 188 |
+
repo_info = get_random_repo()
|
| 189 |
+
if repo_info is None:
|
| 190 |
+
print(f" [{attempt}/{max_attempts}] No repo was obtained. Retrying...")
|
| 191 |
+
continue
|
| 192 |
+
|
| 193 |
+
clone_url = repo_info["clone_url"]
|
| 194 |
+
if clone_url in tried_urls:
|
| 195 |
+
continue
|
| 196 |
+
tried_urls.add(clone_url)
|
| 197 |
+
|
| 198 |
+
tmp_dir = Path(tempfile.mkdtemp(prefix="techdebt_"))
|
| 199 |
+
dest = tmp_dir / repo_info["name"]
|
| 200 |
+
|
| 201 |
+
try:
|
| 202 |
+
print(f" [{attempt}/{max_attempts}] Cloning {repo_info['full_name']} ...")
|
| 203 |
+
repo_git = Repo.clone_from(clone_url, dest)
|
| 204 |
+
except Exception as e:
|
| 205 |
+
print(f" Clone failed: {e}")
|
| 206 |
+
_delete_tree(tmp_dir)
|
| 207 |
+
continue
|
| 208 |
+
|
| 209 |
+
files = find_python_files(dest)
|
| 210 |
+
if not files:
|
| 211 |
+
print(" The repo has no usable .py files. Trying another one...")
|
| 212 |
+
repo_git.close()
|
| 213 |
+
_delete_tree(tmp_dir)
|
| 214 |
+
continue
|
| 215 |
+
|
| 216 |
+
return tmp_dir, dest, repo_git, repo_info["name"], files
|
| 217 |
+
|
| 218 |
+
return None
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
def main():
|
| 222 |
+
parser = argparse.ArgumentParser(
|
| 223 |
+
description="Analyzes a repository and predicts the technical debt per file. "
|
| 224 |
+
"Without arguments, it looks for and clones a random GitHub repo."
|
| 225 |
+
)
|
| 226 |
+
parser.add_argument(
|
| 227 |
+
"repo_path", nargs="?", default=None,
|
| 228 |
+
help="Path to a local repo (optional). If omitted, a random GitHub repo "
|
| 229 |
+
"is looked up and cloned.",
|
| 230 |
+
)
|
| 231 |
+
args = parser.parse_args()
|
| 232 |
+
|
| 233 |
+
tmp_dir = None
|
| 234 |
+
repo_git = None
|
| 235 |
+
try:
|
| 236 |
+
if args.repo_path:
|
| 237 |
+
repo_path, repo_git = validate_repo(args.repo_path)
|
| 238 |
+
repo_name = repo_path.name
|
| 239 |
+
files = find_python_files(repo_path)
|
| 240 |
+
if not files:
|
| 241 |
+
print(f"No .py files were found in '{repo_path}'.")
|
| 242 |
+
return
|
| 243 |
+
else:
|
| 244 |
+
print("Looking for a random repo on GitHub...")
|
| 245 |
+
result = fetch_and_clone_repo()
|
| 246 |
+
if result is None:
|
| 247 |
+
print("Could not obtain a valid GitHub repo after several attempts.")
|
| 248 |
+
return
|
| 249 |
+
tmp_dir, repo_path, repo_git, repo_name, files = result
|
| 250 |
+
|
| 251 |
+
print_summary(repo_path, files)
|
| 252 |
+
|
| 253 |
+
results, functions_list, index_list = feature_extraction(
|
| 254 |
+
repo_path, files, repo_git, repo_name
|
| 255 |
+
)
|
| 256 |
+
print(f"\nFeatures extracted from {len(results)} files.")
|
| 257 |
+
|
| 258 |
+
if not results:
|
| 259 |
+
print("Could not extract features from any file.")
|
| 260 |
+
return
|
| 261 |
+
|
| 262 |
+
embeddings_por_archivo = generar_embeddings_reducidos(functions_list, index_list)
|
| 263 |
+
report = full_report(results, embeddings_por_archivo)
|
| 264 |
+
|
| 265 |
+
print("\n" + "=" * 60)
|
| 266 |
+
print(" RISK + RAW VALUES (highest to lowest risk)")
|
| 267 |
+
print("=" * 60)
|
| 268 |
+
print(report.to_string(index=False))
|
| 269 |
+
|
| 270 |
+
csv_path = save_csv_report(report, repo_name)
|
| 271 |
+
print(f"\nReport saved to: {csv_path}")
|
| 272 |
+
finally:
|
| 273 |
+
if repo_git is not None:
|
| 274 |
+
repo_git.close()
|
| 275 |
+
if tmp_dir is not None:
|
| 276 |
+
_delete_tree(tmp_dir)
|
| 277 |
+
print("\nTemporary repo deleted.")
|
| 278 |
+
|
| 279 |
+
|
| 280 |
+
if __name__ == "__main__":
|
| 281 |
+
main()
|