File size: 6,010 Bytes
34d3ef3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
import ast
from radon.complexity import cc_visit
from radon import metrics, raw
from datetime import date
import numpy as np

class MyAnalyzer(ast.NodeVisitor):
    def __init__(self):
        self.dependency_counts = 0
        self.max_nesting_depth = 0
        self.current_depth = 0
        self.is_tested = 0

    def visit_Import(self, node):
        self.dependency_counts += 1
        self.generic_visit(node)
    def visit_ImportFrom(self, node):
        self.dependency_counts += 1
        self.generic_visit(node)

    def visit_If(self, node):
        self.current_depth += 1

        if self.current_depth > self.max_nesting_depth:
            self.max_nesting_depth = self.current_depth
        self.generic_visit(node)
        self.current_depth -= 1

    def visit_For(self, node):
        self.current_depth += 1

        if self.current_depth > self.max_nesting_depth:
            self.max_nesting_depth = self.current_depth
        self.generic_visit(node)
        self.current_depth -= 1
    def visit_While(self, node):
        self.current_depth += 1

        if self.current_depth > self.max_nesting_depth:
            self.max_nesting_depth = self.current_depth
        self.generic_visit(node)
        self.current_depth -= 1
    def visit_Try(self, node):
        self.current_depth += 1

        if self.current_depth > self.max_nesting_depth:
            self.max_nesting_depth = self.current_depth
        self.generic_visit(node)
        self.current_depth -= 1

    def visit_FunctionDef(self, node):
        self.current_depth += 1
        if self.current_depth > self.max_nesting_depth:
            self.max_nesting_depth = self.current_depth

        if node.name.startswith("test_"):
            self.is_tested = 1
        self.generic_visit(node)
        self.current_depth -= 1


    def visit_ClassDef(self, node):
        self.current_depth += 1

        if self.current_depth > self.max_nesting_depth:
            self.max_nesting_depth = self.current_depth
        if node.name.startswith("Test"):
            self.is_tested = 1
        self.generic_visit(node)
        self.current_depth -= 1


def features_radon(file):
    result_raw_loc = raw.analyze(file).loc  # feat
    result_raw_sloc = raw.analyze(file).sloc  # feat
    result_raw_comments = raw.analyze(file).comments
    result_raw_multi = raw.analyze(file).multi
    result_raw_blank = raw.analyze(file).blank

    comment_ratio = (
        (result_raw_comments + result_raw_multi) / result_raw_loc
        if result_raw_comments > 0 else 0
    )
    blank_ratio = result_raw_blank / result_raw_loc if result_raw_loc > 0 else 0
    # Cyclomatic complexity
    result_cc = cc_visit(file)
    if len(result_cc) > 0:

        sum_func_lines = 0
        complexities = [score.complexity for score in result_cc]
        max_complexity = max(complexities)  # feat
        average_cc = sum(complexities) / len(complexities)  # feat
        for score in result_cc:
            func_lines = score.endline - score.lineno + 1
            sum_func_lines += func_lines

        num_functions = len(result_cc)  # feat

        avg_function_len = sum_func_lines / num_functions  # feat
    else:
        num_functions = 0
        avg_function_len = 0
        average_cc = 0
        max_complexity = 0

    # Maintainability Index
    result_mi = metrics.mi_visit(file, multi=True)  # feat

    # Halstead metrics (effort specifically)
    result_hl = metrics.h_visit(file).total.effort  # feat

    return {
        "loc": result_raw_loc,
        "sloc": result_raw_sloc,
        "comment_ratio": comment_ratio,
        "blank_ratio": blank_ratio,
        "cyclomatic_complexity": average_cc,
        "max_complexity": max_complexity,
        "maintainability_index": result_mi,
        "halstead_effort": result_hl,
        "num_functions": num_functions,
        "avg_function_len": avg_function_len,
    }

def features_ats(file):
    # ast dependency counts and max nesting depth
    result_ast = ast.parse(file)
    analyze = MyAnalyzer()
    analyze.visit(result_ast)
    max_nesting_depth = analyze.max_nesting_depth  # feat
    dependency_counts = analyze.dependency_counts #feat
    nesting_depth = analyze.current_depth
    is_tested = analyze.is_tested  # feat

    return {
        "max_nesting_depth": max_nesting_depth,
        "is_tested": is_tested,
        "dependency_count": dependency_counts
    }



def feature_git_raw(file, repo):
    author_list = set()
    churn_rate_total = 0

    file_commits = list(repo.iter_commits(paths=file))
    commit_count = len(file_commits)

    for commit in file_commits:
        author_list.add(commit.author.name)
        churn_rate = commit.stats.files
        change_file = file.replace("\\", "/")
        if change_file in churn_rate:
            churn_rate_total += (
                    churn_rate[change_file]['deletions']
                    + churn_rate[change_file]['insertions']
            )

    author_count = len(author_list)

    if commit_count > 0:
        days_since_change = (date.today() - file_commits[0].committed_datetime.date()).days
    else:
        days_since_change = 0

    return {
        "commit_count": commit_count,
        "author_count": author_count,
        "days_since_change": days_since_change,
        "churn_rate": churn_rate_total,
    }

def normalization_relative_to_repo(df_features, repo_col="repo"):
    relative_cols = ["commit_count", "author_count", "churn_rate", "days_since_change"]
    for col in relative_cols:
        medians = df_features.groupby(repo_col)[col].transform("median")
        medians = medians.replace(0, 1)
        df_features[col] = df_features[col] / medians
    return df_features

def embedding_extract_functions(code, min_lines=2):
    functions = cc_visit(code)
    lines = code.splitlines()
    texts = []
    for f in functions:
        body = "\n".join(lines[f.lineno - 1 : f.endline])
        if len(body.splitlines()) >= min_lines:
            texts.append(body)
    return texts