cpp-ai-code-detector / feature_extractor.py
nguyenvannhi224's picture
Deploy C++ AI Code Detector Pro Gradio Space
374f3ad
Raw History Blame Contribute Delete
13.6 kB
import re
import math
import lizard
import numpy as np
from collections import Counter
def strip_metadata_headers(code_text: str) -> str:
"""Xóa các khối header/metadata (ví dụ: JSON metadata) ở đầu file nếu có."""
code_text = re.sub(r'^\s*\{.*?"code"\s*:\s*".*?"\s*\}\s*', '', code_text, flags=re.DOTALL)
lines = code_text.split('\n')
cleaned_lines = []
header_passed = False
for line in lines:
if not header_passed:
if line.strip().startswith('//') and any(kw in line.upper() for kw in ['DATASET', 'MODEL', 'GEMINI', 'GPT', 'CATEGORY']):
continue
if line.strip() != '':
header_passed = True
cleaned_lines.append(line)
return "\n".join(cleaned_lines).strip()
class CppFeatureExtractorV8:
"""
Trích xuất 44 đặc trưng tĩnh từ mã nguồn C++.
Sử dụng regex và thư viện lizard để đánh giá complexity.
"""
def __init__(self):
self.feature_names = [
'comment_ratio', 'empty_line_ratio', 'emoji_marker_score', 'avg_line_length', 'max_line_length',
'tab_vs_space_ratio', 'trailing_space_ratio', 'brace_style_consistency', 'avg_identifier_length',
'identifier_length_variance', 'single_char_var_ratio', 'unique_identifier_ratio', 'keyword_to_identifier_ratio',
'avg_cyclomatic_complexity', 'num_functions', 'avg_function_loc', 'halstead_volume',
'halstead_difficulty', 'halstead_effort', 'halstead_bugs', 'maintainability_index',
'code_to_comment_ratio', 'max_nesting_depth', 'total_includes', 'has_bits_stdc',
'macro_count', 'modern_cpp_ratio', 'const_usage_ratio', 'has_fast_io', 'newline_style_ratio',
'shannon_entropy', 'bigram_entropy', 'whitespace_entropy', 'class_count', 'struct_count',
'has_inheritance', 'access_specifier_ratio', 'virtual_override_ratio', 'using_std_ratio',
'try_catch_ratio', 'raw_pointer_ratio', 'getter_setter_ratio', 'std_prefix_ratio',
'cpp_cast_ratio'
]
self.string_pattern = re.compile(r'".*?(?<!\\)"|\'.*?(?<!\\)\'')
self.comment_line_pattern = re.compile(r'//.*')
self.comment_block_pattern = re.compile(r'/\*.*?\*/', re.DOTALL)
self.include_pattern = re.compile(r'#include\s*[<"].*[>"]')
self.macro_pattern = re.compile(r'#define\s+')
self.bits_stdc_pattern = re.compile(r'bits/stdc\+\+\.h')
self.fast_io_pattern = re.compile(r'ios_base::sync_with_stdio|cin\.tie')
self.modern_cpp_pattern = re.compile(r'\b(auto|nullptr|unique_ptr|shared_ptr|constexpr|lambda)\b')
self.const_pattern = re.compile(r'\b(const|constexpr)\b')
self.exception_pattern = re.compile(r'\b(try|catch|throw)\b')
self.single_char_var_pattern = re.compile(r'\b[a-zA-Z]\b')
self.emote_tokens = [
":)", ":-)", ":D", ":-D", ";)", ";-)", "<3",
"^_^", "T_T", "xD", "XD", "UwU", "uwu", "O_O", ":3",
"¯\\_(ツ)_/¯"
]
self.operators_pattern = re.compile(r'(\+|-|\*|/|%|=|==|!=|<|>|<=|>=|&&|\|\||!|&|\||\^|~|<<|>>|\+\+|--|->|\.|::|\?|:)')
self.keywords = {'int', 'void', 'if', 'else', 'while', 'for', 'return', 'class', 'public', 'private', 'struct', 'bool', 'char', 'float', 'double', 'std', 'cout', 'cin', 'endl', 'break', 'continue'}
self.class_pattern = re.compile(r'\bclass\s+[a-zA-Z_]\w*')
self.struct_pattern = re.compile(r'\bstruct\s+[a-zA-Z_]\w*')
self.inheritance_pattern = re.compile(r'\bclass\s+[a-zA-Z_]\w*\s*:\s*(public|private|protected)\b')
self.access_specifier_pattern = re.compile(r'\b(public|private|protected)\s*:')
self.virtual_override_pattern = re.compile(r'\b(virtual|override)\b')
self.using_std_pattern = re.compile(r'\busing\s+namespace\s+std\s*;')
self.try_catch_pattern = re.compile(r'\b(try|catch)\b')
self.raw_pointer_pattern = re.compile(r'\b(int|float|double|char|void|[a-zA-Z_]\w*)\s*\*+\s*[a-zA-Z_]\w*')
self.getter_setter_pattern = re.compile(r'\b(get|set)[A-Z]\w*\b')
self.cpp_cast_pattern = re.compile(r'\b(static_cast|dynamic_cast|reinterpret_cast|const_cast)\b')
def calculate_entropy(self, text):
if not text: return 0.0
prob = [float(c) / len(text) for c in dict(Counter(text)).values()]
return -sum(p * math.log2(p) for p in prob)
def calculate_bigram_entropy(self, text):
if len(text) < 2: return 0.0
bigrams = [text[i:i+2] for i in range(len(text)-1)]
prob = [float(c) / len(bigrams) for c in dict(Counter(bigrams)).values()]
return -sum(p * math.log2(p) for p in prob)
def count_emoji_markers(self, text):
count = 0
for char in text:
codepoint = ord(char)
if (0x1F300 <= codepoint <= 0x1FAFF) or (0x2600 <= codepoint <= 0x27BF):
count += 1
for token in self.emote_tokens:
count += text.count(token)
return count
def score_emoji_markers(self, marker_count):
if marker_count <= 0: return 0.0
if marker_count == 1: return 0.65
if marker_count == 2: return 0.9
return 1.0
def calculate_halstead_metrics(self, pure_code, identifiers):
ops = self.operators_pattern.findall(pure_code)
kw_found = [w for w in re.findall(r'\b[a-zA-Z_]\w*\b', pure_code) if w in self.keywords]
all_operators = ops + kw_found
N1 = len(all_operators)
n1 = len(set(all_operators))
strings = self.string_pattern.findall(pure_code)
numbers = re.findall(r'\b\d+(\.\d+)?\b', pure_code)
all_operands = identifiers + strings + numbers
N2 = len(all_operands)
n2 = len(set(all_operands))
n = n1 + n2
N = N1 + N2
Volume = N * math.log2(n) if n > 0 else 0
Difficulty = (n1 / 2) * (N2 / n2) if n2 > 0 else 0
Effort = Difficulty * Volume
Bugs = Volume / 3000
return Volume, Difficulty, Effort, Bugs
def extract(self, code_raw):
features = {}
lines = code_raw.split('\n')
total_chars = len(code_raw)
pure_lines = [l for l in lines if l.strip() and not l.strip().startswith('//')]
total_loc = len(pure_lines) if len(pure_lines) > 0 else 1
line_comments = self.comment_line_pattern.findall(code_raw)
block_comments = self.comment_block_pattern.findall(code_raw)
comments_text = "\n".join(line_comments + block_comments)
string_literals = self.string_pattern.findall(code_raw)
pure_code = self.string_pattern.sub('', code_raw)
pure_code = self.comment_block_pattern.sub('', pure_code)
pure_code = self.comment_line_pattern.sub('', pure_code)
# --- [A] LAYOUT & FORMATTING ---
features['comment_ratio'] = len(comments_text) / total_chars if total_chars > 0 else 0
features['empty_line_ratio'] = sum(1 for line in lines if not line.strip()) / max(1, len(lines))
marker_text = comments_text + "\n" + "\n".join(string_literals)
emoji_marker_count = self.count_emoji_markers(marker_text)
features['emoji_marker_score'] = self.score_emoji_markers(emoji_marker_count)
line_lengths = [len(l.strip()) for l in pure_lines]
features['avg_line_length'] = np.mean(line_lengths) if line_lengths else 0
features['max_line_length'] = np.max(line_lengths) if line_lengths else 0
spaces = code_raw.count(' ')
tabs = code_raw.count('\t')
features['tab_vs_space_ratio'] = tabs / (spaces + tabs) if (spaces + tabs) > 0 else 0
features['trailing_space_ratio'] = sum(1 for l in lines if l.endswith(' ') or l.endswith('\t')) / max(1, len(lines))
kr_brace = len(re.findall(r'\S\s*\{', pure_code))
allman_brace = len(re.findall(r'^\s*\{', pure_code, re.MULTILINE))
total_braces = kr_brace + allman_brace
allman_ratio = allman_brace / total_braces if total_braces > 0 else 0.5
features['brace_style_consistency'] = abs(allman_ratio - 0.5) * 2
# --- [B] NAMING CONVENTIONS ---
identifiers = re.findall(r'\b[a-zA-Z_][a-zA-Z0-9_]*\b', pure_code)
custom_ids = [w for w in identifiers if w not in self.keywords]
if custom_ids:
id_lengths = [len(w) for w in custom_ids]
features['avg_identifier_length'] = np.mean(id_lengths)
features['identifier_length_variance'] = np.var(id_lengths)
features['single_char_var_ratio'] = len(self.single_char_var_pattern.findall(pure_code)) / len(custom_ids)
features['unique_identifier_ratio'] = len(set(custom_ids)) / len(custom_ids)
else:
features['avg_identifier_length'] = 0
features['identifier_length_variance'] = 0
features['single_char_var_ratio'] = 0
features['unique_identifier_ratio'] = 0
features['keyword_to_identifier_ratio'] = len([w for w in identifiers if w in self.keywords]) / max(1, len(custom_ids))
# --- [C] STRUCTURAL COMPLEXITY ---
try:
analysis = lizard.analyze_source_code("test.cpp", code_raw)
if analysis.function_list:
cc_list = [f.cyclomatic_complexity for f in analysis.function_list]
features['avg_cyclomatic_complexity'] = np.mean(cc_list)
features['num_functions'] = len(analysis.function_list)
features['avg_function_loc'] = np.mean([f.end_line - f.start_line for f in analysis.function_list])
else:
features['avg_cyclomatic_complexity'] = 1.0
features['num_functions'] = 0
features['avg_function_loc'] = 0
except Exception:
features['avg_cyclomatic_complexity'] = 1.0
features['num_functions'] = 0
features['avg_function_loc'] = 0
V, D, E, B = self.calculate_halstead_metrics(pure_code, custom_ids)
features['halstead_volume'] = V
features['halstead_difficulty'] = D
features['halstead_effort'] = E
features['halstead_bugs'] = B
MI = 171 - 5.2 * math.log(max(1, V)) - 0.23 * features['avg_cyclomatic_complexity'] - 16.2 * math.log(max(1, total_loc))
features['maintainability_index'] = max(0, MI)
features['code_to_comment_ratio'] = total_loc / max(1, len(comments_text.split('\n')))
depth, max_depth = 0, 0
for char in pure_code:
if char == '{': depth += 1; max_depth = max(max_depth, depth)
elif char == '}': depth = max(0, depth - 1)
features['max_nesting_depth'] = max_depth
# --- [D] CODING HABITS & IDIOMS ---
features['total_includes'] = len(self.include_pattern.findall(code_raw))
features['has_bits_stdc'] = 1 if self.bits_stdc_pattern.search(code_raw) else 0
features['macro_count'] = len(self.macro_pattern.findall(code_raw))
total_words = len(re.findall(r'\b\w+\b', pure_code))
features['modern_cpp_ratio'] = len(self.modern_cpp_pattern.findall(pure_code)) / max(1, total_words)
features['const_usage_ratio'] = len(self.const_pattern.findall(pure_code)) / max(1, total_words)
features['has_fast_io'] = 1 if self.fast_io_pattern.search(pure_code) else 0
count_endl = pure_code.count('endl')
count_n = pure_code.count('\\n')
features['newline_style_ratio'] = count_n / max(1, (count_n + count_endl))
# --- [E] INFORMATION THEORY ---
features['shannon_entropy'] = self.calculate_entropy(pure_code)
features['bigram_entropy'] = self.calculate_bigram_entropy(pure_code)
spaces_pattern = "".join(['S' if c == ' ' else 'T' if c == '\t' else 'N' if c == '\n' else '' for c in code_raw])
features['whitespace_entropy'] = self.calculate_entropy(spaces_pattern)
# --- [F] NEW OOP & ADVANCED CODING HABITS ---
class_cnt = len(self.class_pattern.findall(pure_code))
features['class_count'] = class_cnt
features['struct_count'] = len(self.struct_pattern.findall(pure_code))
features['has_inheritance'] = 1 if self.inheritance_pattern.search(pure_code) else 0
access_spec_cnt = len(self.access_specifier_pattern.findall(pure_code))
features['access_specifier_ratio'] = access_spec_cnt / max(1, class_cnt)
virtual_override_cnt = len(self.virtual_override_pattern.findall(pure_code))
features['virtual_override_ratio'] = virtual_override_cnt / max(1, features['num_functions'])
features['using_std_ratio'] = 1 if self.using_std_pattern.search(code_raw) else 0
try_catch_cnt = len(self.try_catch_pattern.findall(pure_code))
features['try_catch_ratio'] = try_catch_cnt / max(1, total_loc)
raw_ptr_cnt = len(self.raw_pointer_pattern.findall(pure_code))
features['raw_pointer_ratio'] = raw_ptr_cnt / max(1, total_words)
getter_setter_cnt = len(self.getter_setter_pattern.findall(pure_code))
features['getter_setter_ratio'] = getter_setter_cnt / max(1, len(custom_ids) if custom_ids else 1)
std_prefix_cnt = pure_code.count('std::')
features['std_prefix_ratio'] = std_prefix_cnt / max(1, total_words)
cpp_cast_cnt = len(self.cpp_cast_pattern.findall(pure_code))
features['cpp_cast_ratio'] = cpp_cast_cnt / max(1, total_words)
return {k: features.get(k, 0.0) for k in self.feature_names}