Spaces:
Sleeping
Sleeping
Download src/feature_lib.py from ChilliPeep/techdebt-ml: direct link, hf CLI and curl.
- Browser
- Download file 6.01 kB
-
https://huggingface.co/spaces/ChilliPeep/techdebt-ml/resolve/main/src/feature_lib.py
- Command line
-
hf download hf://spaces/ChilliPeep/techdebt-ml/src/feature_lib.py
-
curl -L -o feature_lib.py https://huggingface.co/spaces/ChilliPeep/techdebt-ml/resolve/main/src/feature_lib.py
6.01 kB
| import ast | |
| from radon.complexity import cc_visit | |
| from radon import metrics, raw | |
| from datetime import date | |
| import numpy as np | |
| class MyAnalyzer(ast.NodeVisitor): | |
| def __init__(self): | |
| self.dependency_counts = 0 | |
| self.max_nesting_depth = 0 | |
| self.current_depth = 0 | |
| self.is_tested = 0 | |
| def visit_Import(self, node): | |
| self.dependency_counts += 1 | |
| self.generic_visit(node) | |
| def visit_ImportFrom(self, node): | |
| self.dependency_counts += 1 | |
| self.generic_visit(node) | |
| def visit_If(self, node): | |
| self.current_depth += 1 | |
| if self.current_depth > self.max_nesting_depth: | |
| self.max_nesting_depth = self.current_depth | |
| self.generic_visit(node) | |
| self.current_depth -= 1 | |
| def visit_For(self, node): | |
| self.current_depth += 1 | |
| if self.current_depth > self.max_nesting_depth: | |
| self.max_nesting_depth = self.current_depth | |
| self.generic_visit(node) | |
| self.current_depth -= 1 | |
| def visit_While(self, node): | |
| self.current_depth += 1 | |
| if self.current_depth > self.max_nesting_depth: | |
| self.max_nesting_depth = self.current_depth | |
| self.generic_visit(node) | |
| self.current_depth -= 1 | |
| def visit_Try(self, node): | |
| self.current_depth += 1 | |
| if self.current_depth > self.max_nesting_depth: | |
| self.max_nesting_depth = self.current_depth | |
| self.generic_visit(node) | |
| self.current_depth -= 1 | |
| def visit_FunctionDef(self, node): | |
| self.current_depth += 1 | |
| if self.current_depth > self.max_nesting_depth: | |
| self.max_nesting_depth = self.current_depth | |
| if node.name.startswith("test_"): | |
| self.is_tested = 1 | |
| self.generic_visit(node) | |
| self.current_depth -= 1 | |
| def visit_ClassDef(self, node): | |
| self.current_depth += 1 | |
| if self.current_depth > self.max_nesting_depth: | |
| self.max_nesting_depth = self.current_depth | |
| if node.name.startswith("Test"): | |
| self.is_tested = 1 | |
| self.generic_visit(node) | |
| self.current_depth -= 1 | |
| def features_radon(file): | |
| result_raw_loc = raw.analyze(file).loc # feat | |
| result_raw_sloc = raw.analyze(file).sloc # feat | |
| result_raw_comments = raw.analyze(file).comments | |
| result_raw_multi = raw.analyze(file).multi | |
| result_raw_blank = raw.analyze(file).blank | |
| comment_ratio = ( | |
| (result_raw_comments + result_raw_multi) / result_raw_loc | |
| if result_raw_comments > 0 else 0 | |
| ) | |
| blank_ratio = result_raw_blank / result_raw_loc if result_raw_loc > 0 else 0 | |
| # Cyclomatic complexity | |
| result_cc = cc_visit(file) | |
| if len(result_cc) > 0: | |
| sum_func_lines = 0 | |
| complexities = [score.complexity for score in result_cc] | |
| max_complexity = max(complexities) # feat | |
| average_cc = sum(complexities) / len(complexities) # feat | |
| for score in result_cc: | |
| func_lines = score.endline - score.lineno + 1 | |
| sum_func_lines += func_lines | |
| num_functions = len(result_cc) # feat | |
| avg_function_len = sum_func_lines / num_functions # feat | |
| else: | |
| num_functions = 0 | |
| avg_function_len = 0 | |
| average_cc = 0 | |
| max_complexity = 0 | |
| # Maintainability Index | |
| result_mi = metrics.mi_visit(file, multi=True) # feat | |
| # Halstead metrics (effort specifically) | |
| result_hl = metrics.h_visit(file).total.effort # feat | |
| return { | |
| "loc": result_raw_loc, | |
| "sloc": result_raw_sloc, | |
| "comment_ratio": comment_ratio, | |
| "blank_ratio": blank_ratio, | |
| "cyclomatic_complexity": average_cc, | |
| "max_complexity": max_complexity, | |
| "maintainability_index": result_mi, | |
| "halstead_effort": result_hl, | |
| "num_functions": num_functions, | |
| "avg_function_len": avg_function_len, | |
| } | |
| def features_ats(file): | |
| # ast dependency counts and max nesting depth | |
| result_ast = ast.parse(file) | |
| analyze = MyAnalyzer() | |
| analyze.visit(result_ast) | |
| max_nesting_depth = analyze.max_nesting_depth # feat | |
| dependency_counts = analyze.dependency_counts #feat | |
| nesting_depth = analyze.current_depth | |
| is_tested = analyze.is_tested # feat | |
| return { | |
| "max_nesting_depth": max_nesting_depth, | |
| "is_tested": is_tested, | |
| "dependency_count": dependency_counts | |
| } | |
| def feature_git_raw(file, repo): | |
| author_list = set() | |
| churn_rate_total = 0 | |
| file_commits = list(repo.iter_commits(paths=file)) | |
| commit_count = len(file_commits) | |
| for commit in file_commits: | |
| author_list.add(commit.author.name) | |
| churn_rate = commit.stats.files | |
| change_file = file.replace("\\", "/") | |
| if change_file in churn_rate: | |
| churn_rate_total += ( | |
| churn_rate[change_file]['deletions'] | |
| + churn_rate[change_file]['insertions'] | |
| ) | |
| author_count = len(author_list) | |
| if commit_count > 0: | |
| days_since_change = (date.today() - file_commits[0].committed_datetime.date()).days | |
| else: | |
| days_since_change = 0 | |
| return { | |
| "commit_count": commit_count, | |
| "author_count": author_count, | |
| "days_since_change": days_since_change, | |
| "churn_rate": churn_rate_total, | |
| } | |
| def normalization_relative_to_repo(df_features, repo_col="repo"): | |
| relative_cols = ["commit_count", "author_count", "churn_rate", "days_since_change"] | |
| for col in relative_cols: | |
| medians = df_features.groupby(repo_col)[col].transform("median") | |
| medians = medians.replace(0, 1) | |
| df_features[col] = df_features[col] / medians | |
| return df_features | |
| def embedding_extract_functions(code, min_lines=2): | |
| functions = cc_visit(code) | |
| lines = code.splitlines() | |
| texts = [] | |
| for f in functions: | |
| body = "\n".join(lines[f.lineno - 1 : f.endline]) | |
| if len(body.splitlines()) >= min_lines: | |
| texts.append(body) | |
| return texts |