Spaces:
Sleeping
Sleeping
File size: 6,010 Bytes
34d3ef3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 | import ast
from radon.complexity import cc_visit
from radon import metrics, raw
from datetime import date
import numpy as np
class MyAnalyzer(ast.NodeVisitor):
def __init__(self):
self.dependency_counts = 0
self.max_nesting_depth = 0
self.current_depth = 0
self.is_tested = 0
def visit_Import(self, node):
self.dependency_counts += 1
self.generic_visit(node)
def visit_ImportFrom(self, node):
self.dependency_counts += 1
self.generic_visit(node)
def visit_If(self, node):
self.current_depth += 1
if self.current_depth > self.max_nesting_depth:
self.max_nesting_depth = self.current_depth
self.generic_visit(node)
self.current_depth -= 1
def visit_For(self, node):
self.current_depth += 1
if self.current_depth > self.max_nesting_depth:
self.max_nesting_depth = self.current_depth
self.generic_visit(node)
self.current_depth -= 1
def visit_While(self, node):
self.current_depth += 1
if self.current_depth > self.max_nesting_depth:
self.max_nesting_depth = self.current_depth
self.generic_visit(node)
self.current_depth -= 1
def visit_Try(self, node):
self.current_depth += 1
if self.current_depth > self.max_nesting_depth:
self.max_nesting_depth = self.current_depth
self.generic_visit(node)
self.current_depth -= 1
def visit_FunctionDef(self, node):
self.current_depth += 1
if self.current_depth > self.max_nesting_depth:
self.max_nesting_depth = self.current_depth
if node.name.startswith("test_"):
self.is_tested = 1
self.generic_visit(node)
self.current_depth -= 1
def visit_ClassDef(self, node):
self.current_depth += 1
if self.current_depth > self.max_nesting_depth:
self.max_nesting_depth = self.current_depth
if node.name.startswith("Test"):
self.is_tested = 1
self.generic_visit(node)
self.current_depth -= 1
def features_radon(file):
result_raw_loc = raw.analyze(file).loc # feat
result_raw_sloc = raw.analyze(file).sloc # feat
result_raw_comments = raw.analyze(file).comments
result_raw_multi = raw.analyze(file).multi
result_raw_blank = raw.analyze(file).blank
comment_ratio = (
(result_raw_comments + result_raw_multi) / result_raw_loc
if result_raw_comments > 0 else 0
)
blank_ratio = result_raw_blank / result_raw_loc if result_raw_loc > 0 else 0
# Cyclomatic complexity
result_cc = cc_visit(file)
if len(result_cc) > 0:
sum_func_lines = 0
complexities = [score.complexity for score in result_cc]
max_complexity = max(complexities) # feat
average_cc = sum(complexities) / len(complexities) # feat
for score in result_cc:
func_lines = score.endline - score.lineno + 1
sum_func_lines += func_lines
num_functions = len(result_cc) # feat
avg_function_len = sum_func_lines / num_functions # feat
else:
num_functions = 0
avg_function_len = 0
average_cc = 0
max_complexity = 0
# Maintainability Index
result_mi = metrics.mi_visit(file, multi=True) # feat
# Halstead metrics (effort specifically)
result_hl = metrics.h_visit(file).total.effort # feat
return {
"loc": result_raw_loc,
"sloc": result_raw_sloc,
"comment_ratio": comment_ratio,
"blank_ratio": blank_ratio,
"cyclomatic_complexity": average_cc,
"max_complexity": max_complexity,
"maintainability_index": result_mi,
"halstead_effort": result_hl,
"num_functions": num_functions,
"avg_function_len": avg_function_len,
}
def features_ats(file):
# ast dependency counts and max nesting depth
result_ast = ast.parse(file)
analyze = MyAnalyzer()
analyze.visit(result_ast)
max_nesting_depth = analyze.max_nesting_depth # feat
dependency_counts = analyze.dependency_counts #feat
nesting_depth = analyze.current_depth
is_tested = analyze.is_tested # feat
return {
"max_nesting_depth": max_nesting_depth,
"is_tested": is_tested,
"dependency_count": dependency_counts
}
def feature_git_raw(file, repo):
author_list = set()
churn_rate_total = 0
file_commits = list(repo.iter_commits(paths=file))
commit_count = len(file_commits)
for commit in file_commits:
author_list.add(commit.author.name)
churn_rate = commit.stats.files
change_file = file.replace("\\", "/")
if change_file in churn_rate:
churn_rate_total += (
churn_rate[change_file]['deletions']
+ churn_rate[change_file]['insertions']
)
author_count = len(author_list)
if commit_count > 0:
days_since_change = (date.today() - file_commits[0].committed_datetime.date()).days
else:
days_since_change = 0
return {
"commit_count": commit_count,
"author_count": author_count,
"days_since_change": days_since_change,
"churn_rate": churn_rate_total,
}
def normalization_relative_to_repo(df_features, repo_col="repo"):
relative_cols = ["commit_count", "author_count", "churn_rate", "days_since_change"]
for col in relative_cols:
medians = df_features.groupby(repo_col)[col].transform("median")
medians = medians.replace(0, 1)
df_features[col] = df_features[col] / medians
return df_features
def embedding_extract_functions(code, min_lines=2):
functions = cc_visit(code)
lines = code.splitlines()
texts = []
for f in functions:
body = "\n".join(lines[f.lineno - 1 : f.endline])
if len(body.splitlines()) >= min_lines:
texts.append(body)
return texts |