nguyenvannhi224 commited on
Commit
374f3ad
·
0 Parent(s):

Deploy C++ AI Code Detector Pro Gradio Space

Browse files
Files changed (7) hide show
  1. .gitattributes +4 -0
  2. .gitignore +6 -0
  3. README.md +25 -0
  4. app.py +327 -0
  5. feature_extractor.py +271 -0
  6. html_renderer.py +109 -0
  7. requirements.txt +11 -0
.gitattributes ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
2
+ *.bin filter=lfs diff=lfs merge=lfs -text
3
+ *.pkl filter=lfs diff=lfs merge=lfs -text
4
+ *.npy filter=lfs diff=lfs merge=lfs -text
.gitignore ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ __pycache__/
2
+ *.pyc
3
+ .env
4
+ local_models/
5
+ DEPLOY_TO_HF.md
6
+ push_to_space.ps1
README.md ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: C++ AI Code Detector Pro
3
+ emoji: 🕵️‍♂️
4
+ colorFrom: indigo
5
+ colorTo: blue
6
+ sdk: gradio
7
+ sdk_version: 6.30.0
8
+ app_file: app.py
9
+ pinned: false
10
+ license: mit
11
+ ---
12
+
13
+ # 🕵️‍♂️ C++ AI Code Detector Pro
14
+
15
+ Hệ thống phát hiện mã nguồn C++ do AI sinh ra (AI Generated vs Human Written) sử dụng kiến trúc lai **Hybrid AI**:
16
+ - **Deep Learning:** GraphCodeBERT / RoBERTa (Ensemble & Single-Fold) với cơ chế giải thích gán nhãn token **Layer Integrated Gradients (LIG)**.
17
+ - **Machine Learning & Feature Engineering:** 44 đặc trưng cú pháp, cấu trúc và thói quen lập trình C++ kết hợp **LightGBM** và **SHAP values**.
18
+ - **Adaptive Controlled Fusion:** Cơ chế hợp nhất thích ứng có kiểm soát giúp tối đa hóa độ chính xác và xử lý vùng biên phân loại.
19
+
20
+ ---
21
+
22
+ ### 🚀 Cách vận hành
23
+ 1. Chọn hoặc dán mã nguồn C++ cần kiểm tra vào khung nhập liệu.
24
+ 2. Hệ thống sẽ tự động phân loại hình thái (OOP vs Procedural) và kích hoạt mô hình phù hợp.
25
+ 3. Bấm **🚀 PHÂN TÍCH NGAY** để nhận kết quả phân tích thời gian thực!
app.py ADDED
@@ -0,0 +1,327 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import sys
3
+ import time
4
+ import math
5
+ import joblib
6
+ import numpy as np
7
+ import pandas as pd
8
+ import torch
9
+ import torch.nn.functional as F
10
+ from transformers import RobertaTokenizer, RobertaForSequenceClassification
11
+ from captum.attr import LayerIntegratedGradients
12
+ import gradio as gr
13
+
14
+ # Try importing Hugging Face ZeroGPU decorator
15
+ try:
16
+ import spaces
17
+ gpu_decorator = spaces.GPU
18
+ except ImportError:
19
+ def gpu_decorator(func):
20
+ return func
21
+
22
+ BASE_DIR = os.path.dirname(os.path.abspath(__file__))
23
+ if BASE_DIR not in sys.path:
24
+ sys.path.insert(0, BASE_DIR)
25
+
26
+ # Import local modules
27
+ from feature_extractor import CppFeatureExtractorV8, strip_metadata_headers
28
+ from html_renderer import render_html_heatmap
29
+
30
+ from huggingface_hub import hf_hub_download
31
+
32
+ HF_REPO_ID = "nguyenvannhi224/cpp-code-detector-models"
33
+
34
+ def resolve_model_dir(relative_name: str) -> str:
35
+ candidates = [
36
+ os.path.join(BASE_DIR, "local_models", relative_name),
37
+ os.path.join(BASE_DIR, relative_name),
38
+ os.path.join(os.path.dirname(BASE_DIR), "local_models", relative_name),
39
+ ]
40
+ for c in candidates:
41
+ if os.path.exists(c):
42
+ return c
43
+ return candidates[0]
44
+
45
+ def get_model_path_or_hub(subfolder: str):
46
+ local_p = resolve_model_dir(subfolder)
47
+ if os.path.isdir(local_p) and os.path.exists(os.path.join(local_p, "model.safetensors")):
48
+ return local_p, None
49
+ return HF_REPO_ID, subfolder
50
+
51
+ def get_pkl_path_or_hub(filename: str) -> str:
52
+ local_p = resolve_model_dir(os.path.join("saved_models", filename))
53
+ if os.path.exists(local_p):
54
+ return local_p
55
+ try:
56
+ return hf_hub_download(repo_id=HF_REPO_ID, filename=f"saved_models/{filename}")
57
+ except Exception:
58
+ return ""
59
+
60
+ # ==============================================================================
61
+ # LOAD MODELS ONCE (GLOBAL)
62
+ # ==============================================================================
63
+ print("⏳ Loading RoBERTa models...")
64
+ path_oop, sub_oop = get_model_path_or_hub("fold_1_oop")
65
+ path_normal, sub_normal = get_model_path_or_hub("fold_1_basic")
66
+
67
+ DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
68
+
69
+ try:
70
+ if sub_oop:
71
+ tokenizer = RobertaTokenizer.from_pretrained(path_oop, subfolder=sub_oop)
72
+ model_oop = RobertaForSequenceClassification.from_pretrained(path_oop, subfolder=sub_oop).to(DEVICE)
73
+ else:
74
+ tokenizer = RobertaTokenizer.from_pretrained(path_oop)
75
+ model_oop = RobertaForSequenceClassification.from_pretrained(path_oop).to(DEVICE)
76
+ model_oop.eval()
77
+ print("✅ Loaded OOP model")
78
+ except Exception as e:
79
+ tokenizer = RobertaTokenizer.from_pretrained("microsoft/graphcodebert-base")
80
+ model_oop = None
81
+ print(f"⚠️ Failed to load OOP model: {e}")
82
+
83
+ try:
84
+ if sub_normal:
85
+ model_normal = RobertaForSequenceClassification.from_pretrained(path_normal, subfolder=sub_normal).to(DEVICE)
86
+ else:
87
+ model_normal = RobertaForSequenceClassification.from_pretrained(path_normal).to(DEVICE)
88
+ model_normal.eval()
89
+ print("✅ Loaded Normal model")
90
+ except Exception as e:
91
+ model_normal = None
92
+ print(f"⚠️ Failed to load Normal model: {e}")
93
+
94
+ print("⏳ Loading LightGBM & Scaler...")
95
+ extractor = CppFeatureExtractorV8()
96
+
97
+ model_path = get_pkl_path_or_hub("LightGBM_Regulated.pkl")
98
+ scaler_path = get_pkl_path_or_hub("scaler.pkl")
99
+ feat_path = get_pkl_path_or_hub("final_features.pkl")
100
+ if not feat_path:
101
+ feat_path = get_pkl_path_or_hub("feature_names.pkl")
102
+
103
+ lgbm_model = None
104
+ scaler = None
105
+ feature_names = None
106
+
107
+ if model_path and scaler_path and feat_path:
108
+ try:
109
+ lgbm_model = joblib.load(model_path)
110
+ scaler = joblib.load(scaler_path)
111
+ feature_names = joblib.load(feat_path)
112
+ print("✅ LightGBM pipeline ready")
113
+ except Exception as e:
114
+ print(f"⚠️ LightGBM load failed: {e}")
115
+
116
+ # ==============================================================================
117
+ # INFERENCE LOGIC
118
+ # ==============================================================================
119
+ def classify_oop_or_normal(code: str) -> str:
120
+ oop_keywords = ["class ", "struct ", "public:", "private:", "protected:", "virtual ", "override", "std::unique_ptr"]
121
+ hits = sum(1 for kw in oop_keywords if kw in code)
122
+ return "OOP" if hits >= 2 else "NORMAL"
123
+
124
+ def run_lig_attribution(model, tokenizer, input_ids, target_class=1, n_steps=10):
125
+ embeddings = model.roberta.embeddings
126
+
127
+ def forward_func(inputs_embeds):
128
+ outputs = model(inputs_embeds=inputs_embeds)
129
+ return outputs.logits[:, target_class]
130
+
131
+ lig = LayerIntegratedGradients(forward_func, embeddings)
132
+ input_embeds = embeddings(input_ids)
133
+ baseline_embeds = torch.zeros_like(input_embeds)
134
+
135
+ attributions, _ = lig.attribute(
136
+ inputs=input_embeds,
137
+ baselines=baseline_embeds,
138
+ n_steps=n_steps,
139
+ return_convergence_delta=True
140
+ )
141
+ tokens_attr = attributions.sum(dim=-1).squeeze(0).detach().cpu().numpy()
142
+ tokens = tokenizer.convert_ids_to_tokens(input_ids.squeeze(0).tolist())
143
+ return tokens, tokens_attr
144
+
145
+ @gpu_decorator
146
+ def predict_code(code_text: str, threshold: float = 0.5):
147
+ if not code_text or not code_text.strip():
148
+ return {"Lỗi": "Vui lòng nhập code C++"}, {}, "<p>Chưa có dữ liệu</p>", pd.DataFrame()
149
+
150
+ t0 = time.perf_counter()
151
+ clean_code = strip_metadata_headers(code_text)
152
+
153
+ # 1. Routing
154
+ code_type = classify_oop_or_normal(clean_code)
155
+ active_model = model_oop if code_type == "OOP" and model_oop is not None else model_normal
156
+ model_label = f"C++ {code_type} Model"
157
+
158
+ # 2. Machine Learning
159
+ lgbm_score = 0.5
160
+ extracted_features = {}
161
+ features_df = pd.DataFrame()
162
+ if extractor is not None and lgbm_model is not None and scaler is not None and feature_names is not None:
163
+ extracted_features = extractor.extract(clean_code)
164
+ df = pd.DataFrame([extracted_features]).fillna(0)
165
+ try:
166
+ X_all = df[scaler.feature_names_in_]
167
+ X_scaled = scaler.transform(X_all)
168
+ feat_list = list(feature_names)
169
+ all_feat = list(scaler.feature_names_in_)
170
+ indices = [all_feat.index(f) for f in feat_list]
171
+ X_final = X_scaled[:, indices]
172
+ lgbm_score = float(lgbm_model.predict_proba(X_final)[0][1])
173
+
174
+ # Format dataframe for display
175
+ feat_rows = [{"Đặc trưng": k, "Giá trị": f"{v:.4f}" if isinstance(v, float) else str(v)} for k, v in extracted_features.items()]
176
+ features_df = pd.DataFrame(feat_rows)
177
+ except Exception:
178
+ lgbm_score = 0.5
179
+
180
+ # 3. Deep Learning
181
+ dl_score = 0.5
182
+ html_heatmap = ""
183
+ if active_model and tokenizer:
184
+ cur_device = next(active_model.parameters()).device
185
+ inputs = tokenizer(
186
+ clean_code,
187
+ return_tensors="pt",
188
+ truncation=True,
189
+ max_length=512,
190
+ return_attention_mask=True
191
+ )
192
+ inputs = {k: v.to(cur_device) for k, v in inputs.items()}
193
+ input_ids = inputs["input_ids"]
194
+
195
+ with torch.no_grad():
196
+ logits = active_model(**inputs).logits
197
+ probs = F.softmax(logits, dim=-1)
198
+ dl_score = float(probs[0][1].item())
199
+
200
+ try:
201
+ tokens, attributions = run_lig_attribution(active_model, tokenizer, input_ids, target_class=1, n_steps=10)
202
+ html_heatmap = render_html_heatmap(tokens, attributions, title=f"LIG Attribution — {model_label}")
203
+ except Exception as e:
204
+ html_heatmap = f"<p>Lỗi tạo heatmap: {e}</p>"
205
+
206
+ # 4. Controlled Adaptive Fusion
207
+ final_score = 0.48 * dl_score + 0.52 * lgbm_score
208
+ conflict = False
209
+ if abs(dl_score - lgbm_score) >= 0.40:
210
+ conflict = True
211
+ if 0.40 <= dl_score <= 0.70:
212
+ final_score = 0.70 * dl_score + 0.30 * lgbm_score
213
+
214
+ pred_label = "AI GENERATED" if final_score >= threshold else "HUMAN WRITTEN"
215
+ latency_ms = (time.perf_counter() - t0) * 1000
216
+
217
+ label_output = {
218
+ "AI GENERATED": float(final_score),
219
+ "HUMAN WRITTEN": float(1.0 - final_score),
220
+ }
221
+
222
+ metrics_output = {
223
+ "Kết luận": pred_label,
224
+ "Xác suất AI": f"{final_score * 100:.1f}%",
225
+ "Mô hình định tuyến": model_label,
226
+ "Điểm Deep Learning (GraphCodeBERT)": f"{dl_score:.4f}",
227
+ "Điểm Machine Learning (LightGBM)": f"{lgbm_score:.4f}",
228
+ "Hợp nhất có kiểm soát (Fusion)": "Đã kích hoạt" if conflict else "Chuẩn",
229
+ "Thời gian xử lý": f"{latency_ms:.0f} ms",
230
+ }
231
+
232
+ return label_output, metrics_output, html_heatmap, features_df
233
+
234
+ # ==============================================================================
235
+ # GRADIO INTERFACE
236
+ # ==============================================================================
237
+ example_human = """#include <iostream>
238
+ #include <vector>
239
+ #include <algorithm>
240
+ using namespace std;
241
+
242
+ int main() {
243
+ ios_base::sync_with_stdio(false);
244
+ cin.tie(NULL);
245
+
246
+ int n;
247
+ if (!(cin >> n)) return 0;
248
+ vector<int> a(n);
249
+ for (int i = 0; i < n; i++) cin >> a[i];
250
+
251
+ sort(a.begin(), a.end());
252
+ for (int x : a) cout << x << " ";
253
+ cout << "\\n";
254
+ return 0;
255
+ }"""
256
+
257
+ example_ai = """#include <iostream>
258
+ #include <vector>
259
+ #include <numeric>
260
+ #include <stdexcept>
261
+
262
+ class StatisticsAnalyzer {
263
+ private:
264
+ std::vector<double> dataset;
265
+
266
+ public:
267
+ explicit StatisticsAnalyzer(const std::vector<double>& data) : dataset(data) {
268
+ if (data.empty()) {
269
+ throw std::invalid_argument("Dataset cannot be empty.");
270
+ }
271
+ }
272
+
273
+ double calculateMean() const {
274
+ double sum = std::accumulate(dataset.begin(), dataset.end(), 0.0);
275
+ return sum / dataset.size();
276
+ }
277
+ };
278
+
279
+ int main() {
280
+ try {
281
+ std::vector<double> numbers = {10.5, 20.3, 30.2, 40.8};
282
+ StatisticsAnalyzer analyzer(numbers);
283
+ std::cout << "Mean: " << analyzer.calculateMean() << std::endl;
284
+ } catch (const std::exception& ex) {
285
+ std::cerr << "Error: " << ex.what() << std::endl;
286
+ }
287
+ return 0;
288
+ }"""
289
+
290
+ with gr.Blocks(title="C++ AI Code Detector Pro") as demo:
291
+ gr.Markdown("# 🕵️‍♂️ C++ AI Code Detector Pro")
292
+ gr.Markdown("Hệ thống phát hiện mã nguồn C++ do AI sinh ra kết hợp **GraphCodeBERT**, **LightGBM 44 đặc trưng**, và **Layer Integrated Gradients (LIG) Attribution**.")
293
+
294
+ with gr.Row():
295
+ with gr.Column(scale=5):
296
+ code_input = gr.Code(label="Mã nguồn C++ cần phân tích", language="cpp", lines=16, value=example_human)
297
+ threshold_slider = gr.Slider(0.1, 0.9, value=0.5, step=0.05, label="Ngưỡng phân loại AI (Threshold)")
298
+ analyze_btn = gr.Button("🚀 PHÂN TÍCH NGAY", variant="primary")
299
+
300
+ gr.Examples(
301
+ examples=[
302
+ [example_human, 0.5],
303
+ [example_ai, 0.5],
304
+ ],
305
+ inputs=[code_input, threshold_slider],
306
+ label="📁 Bấm chọn code mẫu để thử nhanh:"
307
+ )
308
+
309
+ with gr.Column(scale=5):
310
+ output_label = gr.Label(label="Kết luận phân loại", num_top_classes=2)
311
+ output_metrics = gr.JSON(label="Chỉ số phân tích chi tiết (Metrics)")
312
+
313
+ with gr.Tabs():
314
+ with gr.TabItem("🎨 Bản đồ nhiệt gán nhãn token (LIG Heatmap)"):
315
+ gr.Markdown("Các token màu **xanh lá** thể hiện tín hiệu AI sinh ra; màu **đỏ** thể hiện thói quen viết tay của con người.")
316
+ output_heatmap = gr.HTML()
317
+ with gr.TabItem("📊 Đặc trưng cú pháp & hình thái (Stylometric Features)"):
318
+ output_features = gr.Dataframe(headers=["Đặc trưng", "Giá trị"], interactive=False)
319
+
320
+ analyze_btn.click(
321
+ fn=predict_code,
322
+ inputs=[code_input, threshold_slider],
323
+ outputs=[output_label, output_metrics, output_heatmap, output_features]
324
+ )
325
+
326
+ if __name__ == "__main__":
327
+ demo.launch()
feature_extractor.py ADDED
@@ -0,0 +1,271 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import re
2
+ import math
3
+ import lizard
4
+ import numpy as np
5
+ from collections import Counter
6
+
7
+
8
+ def strip_metadata_headers(code_text: str) -> str:
9
+ """Xóa các khối header/metadata (ví dụ: JSON metadata) ở đầu file nếu có."""
10
+ code_text = re.sub(r'^\s*\{.*?"code"\s*:\s*".*?"\s*\}\s*', '', code_text, flags=re.DOTALL)
11
+
12
+ lines = code_text.split('\n')
13
+ cleaned_lines = []
14
+ header_passed = False
15
+ for line in lines:
16
+ if not header_passed:
17
+ if line.strip().startswith('//') and any(kw in line.upper() for kw in ['DATASET', 'MODEL', 'GEMINI', 'GPT', 'CATEGORY']):
18
+ continue
19
+ if line.strip() != '':
20
+ header_passed = True
21
+ cleaned_lines.append(line)
22
+ return "\n".join(cleaned_lines).strip()
23
+
24
+
25
+ class CppFeatureExtractorV8:
26
+ """
27
+ Trích xuất 44 đặc trưng tĩnh từ mã nguồn C++.
28
+ Sử dụng regex và thư viện lizard để đánh giá complexity.
29
+ """
30
+ def __init__(self):
31
+ self.feature_names = [
32
+ 'comment_ratio', 'empty_line_ratio', 'emoji_marker_score', 'avg_line_length', 'max_line_length',
33
+ 'tab_vs_space_ratio', 'trailing_space_ratio', 'brace_style_consistency', 'avg_identifier_length',
34
+ 'identifier_length_variance', 'single_char_var_ratio', 'unique_identifier_ratio', 'keyword_to_identifier_ratio',
35
+ 'avg_cyclomatic_complexity', 'num_functions', 'avg_function_loc', 'halstead_volume',
36
+ 'halstead_difficulty', 'halstead_effort', 'halstead_bugs', 'maintainability_index',
37
+ 'code_to_comment_ratio', 'max_nesting_depth', 'total_includes', 'has_bits_stdc',
38
+ 'macro_count', 'modern_cpp_ratio', 'const_usage_ratio', 'has_fast_io', 'newline_style_ratio',
39
+ 'shannon_entropy', 'bigram_entropy', 'whitespace_entropy', 'class_count', 'struct_count',
40
+ 'has_inheritance', 'access_specifier_ratio', 'virtual_override_ratio', 'using_std_ratio',
41
+ 'try_catch_ratio', 'raw_pointer_ratio', 'getter_setter_ratio', 'std_prefix_ratio',
42
+ 'cpp_cast_ratio'
43
+ ]
44
+
45
+ self.string_pattern = re.compile(r'".*?(?<!\\)"|\'.*?(?<!\\)\'')
46
+ self.comment_line_pattern = re.compile(r'//.*')
47
+ self.comment_block_pattern = re.compile(r'/\*.*?\*/', re.DOTALL)
48
+
49
+ self.include_pattern = re.compile(r'#include\s*[<"].*[>"]')
50
+ self.macro_pattern = re.compile(r'#define\s+')
51
+ self.bits_stdc_pattern = re.compile(r'bits/stdc\+\+\.h')
52
+ self.fast_io_pattern = re.compile(r'ios_base::sync_with_stdio|cin\.tie')
53
+
54
+ self.modern_cpp_pattern = re.compile(r'\b(auto|nullptr|unique_ptr|shared_ptr|constexpr|lambda)\b')
55
+ self.const_pattern = re.compile(r'\b(const|constexpr)\b')
56
+ self.exception_pattern = re.compile(r'\b(try|catch|throw)\b')
57
+ self.single_char_var_pattern = re.compile(r'\b[a-zA-Z]\b')
58
+
59
+ self.emote_tokens = [
60
+ ":)", ":-)", ":D", ":-D", ";)", ";-)", "<3",
61
+ "^_^", "T_T", "xD", "XD", "UwU", "uwu", "O_O", ":3",
62
+ "¯\\_(ツ)_/¯"
63
+ ]
64
+
65
+ self.operators_pattern = re.compile(r'(\+|-|\*|/|%|=|==|!=|<|>|<=|>=|&&|\|\||!|&|\||\^|~|<<|>>|\+\+|--|->|\.|::|\?|:)')
66
+ self.keywords = {'int', 'void', 'if', 'else', 'while', 'for', 'return', 'class', 'public', 'private', 'struct', 'bool', 'char', 'float', 'double', 'std', 'cout', 'cin', 'endl', 'break', 'continue'}
67
+
68
+ self.class_pattern = re.compile(r'\bclass\s+[a-zA-Z_]\w*')
69
+ self.struct_pattern = re.compile(r'\bstruct\s+[a-zA-Z_]\w*')
70
+ self.inheritance_pattern = re.compile(r'\bclass\s+[a-zA-Z_]\w*\s*:\s*(public|private|protected)\b')
71
+ self.access_specifier_pattern = re.compile(r'\b(public|private|protected)\s*:')
72
+ self.virtual_override_pattern = re.compile(r'\b(virtual|override)\b')
73
+ self.using_std_pattern = re.compile(r'\busing\s+namespace\s+std\s*;')
74
+ self.try_catch_pattern = re.compile(r'\b(try|catch)\b')
75
+ self.raw_pointer_pattern = re.compile(r'\b(int|float|double|char|void|[a-zA-Z_]\w*)\s*\*+\s*[a-zA-Z_]\w*')
76
+ self.getter_setter_pattern = re.compile(r'\b(get|set)[A-Z]\w*\b')
77
+ self.cpp_cast_pattern = re.compile(r'\b(static_cast|dynamic_cast|reinterpret_cast|const_cast)\b')
78
+
79
+ def calculate_entropy(self, text):
80
+ if not text: return 0.0
81
+ prob = [float(c) / len(text) for c in dict(Counter(text)).values()]
82
+ return -sum(p * math.log2(p) for p in prob)
83
+
84
+ def calculate_bigram_entropy(self, text):
85
+ if len(text) < 2: return 0.0
86
+ bigrams = [text[i:i+2] for i in range(len(text)-1)]
87
+ prob = [float(c) / len(bigrams) for c in dict(Counter(bigrams)).values()]
88
+ return -sum(p * math.log2(p) for p in prob)
89
+
90
+ def count_emoji_markers(self, text):
91
+ count = 0
92
+ for char in text:
93
+ codepoint = ord(char)
94
+ if (0x1F300 <= codepoint <= 0x1FAFF) or (0x2600 <= codepoint <= 0x27BF):
95
+ count += 1
96
+ for token in self.emote_tokens:
97
+ count += text.count(token)
98
+ return count
99
+
100
+ def score_emoji_markers(self, marker_count):
101
+ if marker_count <= 0: return 0.0
102
+ if marker_count == 1: return 0.65
103
+ if marker_count == 2: return 0.9
104
+ return 1.0
105
+
106
+ def calculate_halstead_metrics(self, pure_code, identifiers):
107
+ ops = self.operators_pattern.findall(pure_code)
108
+ kw_found = [w for w in re.findall(r'\b[a-zA-Z_]\w*\b', pure_code) if w in self.keywords]
109
+ all_operators = ops + kw_found
110
+
111
+ N1 = len(all_operators)
112
+ n1 = len(set(all_operators))
113
+
114
+ strings = self.string_pattern.findall(pure_code)
115
+ numbers = re.findall(r'\b\d+(\.\d+)?\b', pure_code)
116
+ all_operands = identifiers + strings + numbers
117
+
118
+ N2 = len(all_operands)
119
+ n2 = len(set(all_operands))
120
+
121
+ n = n1 + n2
122
+ N = N1 + N2
123
+
124
+ Volume = N * math.log2(n) if n > 0 else 0
125
+ Difficulty = (n1 / 2) * (N2 / n2) if n2 > 0 else 0
126
+ Effort = Difficulty * Volume
127
+ Bugs = Volume / 3000
128
+
129
+ return Volume, Difficulty, Effort, Bugs
130
+
131
+ def extract(self, code_raw):
132
+ features = {}
133
+ lines = code_raw.split('\n')
134
+ total_chars = len(code_raw)
135
+ pure_lines = [l for l in lines if l.strip() and not l.strip().startswith('//')]
136
+ total_loc = len(pure_lines) if len(pure_lines) > 0 else 1
137
+
138
+ line_comments = self.comment_line_pattern.findall(code_raw)
139
+ block_comments = self.comment_block_pattern.findall(code_raw)
140
+ comments_text = "\n".join(line_comments + block_comments)
141
+ string_literals = self.string_pattern.findall(code_raw)
142
+
143
+ pure_code = self.string_pattern.sub('', code_raw)
144
+ pure_code = self.comment_block_pattern.sub('', pure_code)
145
+ pure_code = self.comment_line_pattern.sub('', pure_code)
146
+
147
+ # --- [A] LAYOUT & FORMATTING ---
148
+ features['comment_ratio'] = len(comments_text) / total_chars if total_chars > 0 else 0
149
+ features['empty_line_ratio'] = sum(1 for line in lines if not line.strip()) / max(1, len(lines))
150
+ marker_text = comments_text + "\n" + "\n".join(string_literals)
151
+ emoji_marker_count = self.count_emoji_markers(marker_text)
152
+ features['emoji_marker_score'] = self.score_emoji_markers(emoji_marker_count)
153
+
154
+ line_lengths = [len(l.strip()) for l in pure_lines]
155
+ features['avg_line_length'] = np.mean(line_lengths) if line_lengths else 0
156
+ features['max_line_length'] = np.max(line_lengths) if line_lengths else 0
157
+
158
+ spaces = code_raw.count(' ')
159
+ tabs = code_raw.count('\t')
160
+ features['tab_vs_space_ratio'] = tabs / (spaces + tabs) if (spaces + tabs) > 0 else 0
161
+ features['trailing_space_ratio'] = sum(1 for l in lines if l.endswith(' ') or l.endswith('\t')) / max(1, len(lines))
162
+
163
+ kr_brace = len(re.findall(r'\S\s*\{', pure_code))
164
+ allman_brace = len(re.findall(r'^\s*\{', pure_code, re.MULTILINE))
165
+ total_braces = kr_brace + allman_brace
166
+ allman_ratio = allman_brace / total_braces if total_braces > 0 else 0.5
167
+ features['brace_style_consistency'] = abs(allman_ratio - 0.5) * 2
168
+
169
+ # --- [B] NAMING CONVENTIONS ---
170
+ identifiers = re.findall(r'\b[a-zA-Z_][a-zA-Z0-9_]*\b', pure_code)
171
+ custom_ids = [w for w in identifiers if w not in self.keywords]
172
+
173
+ if custom_ids:
174
+ id_lengths = [len(w) for w in custom_ids]
175
+ features['avg_identifier_length'] = np.mean(id_lengths)
176
+ features['identifier_length_variance'] = np.var(id_lengths)
177
+ features['single_char_var_ratio'] = len(self.single_char_var_pattern.findall(pure_code)) / len(custom_ids)
178
+ features['unique_identifier_ratio'] = len(set(custom_ids)) / len(custom_ids)
179
+ else:
180
+ features['avg_identifier_length'] = 0
181
+ features['identifier_length_variance'] = 0
182
+ features['single_char_var_ratio'] = 0
183
+ features['unique_identifier_ratio'] = 0
184
+
185
+ features['keyword_to_identifier_ratio'] = len([w for w in identifiers if w in self.keywords]) / max(1, len(custom_ids))
186
+
187
+ # --- [C] STRUCTURAL COMPLEXITY ---
188
+ try:
189
+ analysis = lizard.analyze_source_code("test.cpp", code_raw)
190
+ if analysis.function_list:
191
+ cc_list = [f.cyclomatic_complexity for f in analysis.function_list]
192
+ features['avg_cyclomatic_complexity'] = np.mean(cc_list)
193
+ features['num_functions'] = len(analysis.function_list)
194
+ features['avg_function_loc'] = np.mean([f.end_line - f.start_line for f in analysis.function_list])
195
+ else:
196
+ features['avg_cyclomatic_complexity'] = 1.0
197
+ features['num_functions'] = 0
198
+ features['avg_function_loc'] = 0
199
+ except Exception:
200
+ features['avg_cyclomatic_complexity'] = 1.0
201
+ features['num_functions'] = 0
202
+ features['avg_function_loc'] = 0
203
+
204
+ V, D, E, B = self.calculate_halstead_metrics(pure_code, custom_ids)
205
+ features['halstead_volume'] = V
206
+ features['halstead_difficulty'] = D
207
+ features['halstead_effort'] = E
208
+ features['halstead_bugs'] = B
209
+
210
+ MI = 171 - 5.2 * math.log(max(1, V)) - 0.23 * features['avg_cyclomatic_complexity'] - 16.2 * math.log(max(1, total_loc))
211
+ features['maintainability_index'] = max(0, MI)
212
+ features['code_to_comment_ratio'] = total_loc / max(1, len(comments_text.split('\n')))
213
+
214
+ depth, max_depth = 0, 0
215
+ for char in pure_code:
216
+ if char == '{': depth += 1; max_depth = max(max_depth, depth)
217
+ elif char == '}': depth = max(0, depth - 1)
218
+ features['max_nesting_depth'] = max_depth
219
+
220
+ # --- [D] CODING HABITS & IDIOMS ---
221
+ features['total_includes'] = len(self.include_pattern.findall(code_raw))
222
+ features['has_bits_stdc'] = 1 if self.bits_stdc_pattern.search(code_raw) else 0
223
+ features['macro_count'] = len(self.macro_pattern.findall(code_raw))
224
+
225
+ total_words = len(re.findall(r'\b\w+\b', pure_code))
226
+ features['modern_cpp_ratio'] = len(self.modern_cpp_pattern.findall(pure_code)) / max(1, total_words)
227
+ features['const_usage_ratio'] = len(self.const_pattern.findall(pure_code)) / max(1, total_words)
228
+
229
+ features['has_fast_io'] = 1 if self.fast_io_pattern.search(pure_code) else 0
230
+
231
+ count_endl = pure_code.count('endl')
232
+ count_n = pure_code.count('\\n')
233
+ features['newline_style_ratio'] = count_n / max(1, (count_n + count_endl))
234
+
235
+ # --- [E] INFORMATION THEORY ---
236
+ features['shannon_entropy'] = self.calculate_entropy(pure_code)
237
+ features['bigram_entropy'] = self.calculate_bigram_entropy(pure_code)
238
+
239
+ spaces_pattern = "".join(['S' if c == ' ' else 'T' if c == '\t' else 'N' if c == '\n' else '' for c in code_raw])
240
+ features['whitespace_entropy'] = self.calculate_entropy(spaces_pattern)
241
+
242
+ # --- [F] NEW OOP & ADVANCED CODING HABITS ---
243
+ class_cnt = len(self.class_pattern.findall(pure_code))
244
+ features['class_count'] = class_cnt
245
+ features['struct_count'] = len(self.struct_pattern.findall(pure_code))
246
+ features['has_inheritance'] = 1 if self.inheritance_pattern.search(pure_code) else 0
247
+
248
+ access_spec_cnt = len(self.access_specifier_pattern.findall(pure_code))
249
+ features['access_specifier_ratio'] = access_spec_cnt / max(1, class_cnt)
250
+
251
+ virtual_override_cnt = len(self.virtual_override_pattern.findall(pure_code))
252
+ features['virtual_override_ratio'] = virtual_override_cnt / max(1, features['num_functions'])
253
+
254
+ features['using_std_ratio'] = 1 if self.using_std_pattern.search(code_raw) else 0
255
+
256
+ try_catch_cnt = len(self.try_catch_pattern.findall(pure_code))
257
+ features['try_catch_ratio'] = try_catch_cnt / max(1, total_loc)
258
+
259
+ raw_ptr_cnt = len(self.raw_pointer_pattern.findall(pure_code))
260
+ features['raw_pointer_ratio'] = raw_ptr_cnt / max(1, total_words)
261
+
262
+ getter_setter_cnt = len(self.getter_setter_pattern.findall(pure_code))
263
+ features['getter_setter_ratio'] = getter_setter_cnt / max(1, len(custom_ids) if custom_ids else 1)
264
+
265
+ std_prefix_cnt = pure_code.count('std::')
266
+ features['std_prefix_ratio'] = std_prefix_cnt / max(1, total_words)
267
+
268
+ cpp_cast_cnt = len(self.cpp_cast_pattern.findall(pure_code))
269
+ features['cpp_cast_ratio'] = cpp_cast_cnt / max(1, total_words)
270
+
271
+ return {k: features.get(k, 0.0) for k in self.feature_names}
html_renderer.py ADDED
@@ -0,0 +1,109 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import html
2
+ import numpy as np
3
+ from collections import defaultdict
4
+
5
+
6
+ def render_html_heatmap(
7
+ tokens: list[str],
8
+ attributions: list[float],
9
+ title: str = "CODE ANALYSIS",
10
+ ) -> str:
11
+ """Render an HTML heatmap visualization of token attributions."""
12
+ if not tokens or not attributions:
13
+ return "<html><body><p>No attribution data available.</p></body></html>"
14
+
15
+ aggregated_blocks = []
16
+ current_tokens = []
17
+ current_score_sum = 0.0
18
+ token_count = 0
19
+
20
+ for token, score in zip(tokens, attributions):
21
+ if token in ['<s>', '</s>', '<pad>']:
22
+ continue
23
+
24
+ is_start_new_word = token.startswith('Ġ') or token.startswith('Ċ')
25
+ if is_start_new_word and current_tokens:
26
+ raw_str = "".join(current_tokens)
27
+ full_word = raw_str.replace("Ġ", " ").replace("Ċ", "\n")
28
+ if full_word:
29
+ avg_score = current_score_sum / token_count if token_count > 0 else 0
30
+ aggregated_blocks.append((full_word, avg_score))
31
+ current_tokens = []
32
+ current_score_sum = 0.0
33
+ token_count = 0
34
+
35
+ current_tokens.append(token)
36
+ current_score_sum += score
37
+ token_count += 1
38
+
39
+ if current_tokens:
40
+ raw_str = "".join(current_tokens)
41
+ full_word = raw_str.replace("Ġ", " ").replace("Ċ", "\n")
42
+ if full_word:
43
+ aggregated_blocks.append((full_word, current_score_sum / token_count))
44
+
45
+ word_stats = defaultdict(list)
46
+ for word, score in aggregated_blocks:
47
+ clean_word = word.strip()
48
+ if clean_word:
49
+ word_stats[clean_word].append(score)
50
+
51
+ unique_words_list = [(word, sum(score_list) / len(score_list)) for word, score_list in word_stats.items()]
52
+ pos_words = sorted([(w, s) for w, s in unique_words_list if s > 0], key=lambda x: x[1], reverse=True)
53
+ neg_words = sorted([(w, s) for w, s in unique_words_list if s < 0], key=lambda x: x[1])
54
+
55
+ top_k = 50
56
+ ai_list_html = [
57
+ f"<tr><td style='color:#2ecc71; font-weight:bold; padding:4px 8px; border-bottom:1px solid #f0f0f0;'>{html.escape(w)}</td><td style='text-align:right; color:#555; padding:4px 8px; border-bottom:1px solid #f0f0f0;'>{s:.3f}</td></tr>"
58
+ for w, s in pos_words[:top_k]
59
+ ]
60
+ human_list_html = [
61
+ f"<tr><td style='color:#e74c3c; font-weight:bold; padding:4px 8px; border-bottom:1px solid #f0f0f0;'>{html.escape(w)}</td><td style='text-align:right; color:#555; padding:4px 8px; border-bottom:1px solid #f0f0f0;'>{s:.3f}</td></tr>"
62
+ for w, s in neg_words[:top_k]
63
+ ]
64
+
65
+ code_html_parts = []
66
+ all_scores = [abs(s) for w, s in aggregated_blocks]
67
+ max_impact = float(np.max(all_scores)) if len(all_scores) > 0 and float(np.max(all_scores)) > 0 else 1.0
68
+
69
+ for word, score in aggregated_blocks:
70
+ safe_word = html.escape(word)
71
+ relative_score = abs(score) / max_impact
72
+ alpha = min(float(np.power(relative_score, 0.35)), 1.0)
73
+ bg_color = "transparent"
74
+ if alpha >= 0.02:
75
+ bg_color = f"rgba(46, 204, 113, {alpha:.2f})" if score > 0 else f"rgba(231, 76, 60, {alpha:.2f})"
76
+ code_html_parts.append(f'<span style="background-color: {bg_color}; border-radius: 2px;">{safe_word}</span>')
77
+
78
+ code_html = "".join(code_html_parts)
79
+
80
+ human_table = "".join(human_list_html) if human_list_html else "<tr><td colspan='2' align='center' style='padding:10px; color:#999'>None</td></tr>"
81
+ ai_table = "".join(ai_list_html) if ai_list_html else "<tr><td colspan='2' align='center' style='padding:10px; color:#999'>None</td></tr>"
82
+
83
+ return f"""
84
+ <html>
85
+ <head><meta charset="utf-8"></head>
86
+ <body style="margin:0; padding:0; background-color: #f8f9fa;">
87
+ <div style="display: flex; gap: 20px; font-family: 'Segoe UI', sans-serif; margin-bottom: 30px;">
88
+ <div style="flex: 3; border: 1px solid #ccc; border-radius: 8px; background-color: #fff; overflow: hidden; box-shadow: 0 4px 10px rgba(0,0,0,0.08);">
89
+ <div style="background-color: #2c3e50; padding: 12px 15px; border-bottom: 1px solid #000; font-weight: bold; color: #ecf0f1; font-size: 14px;">📑 {html.escape(title)}</div>
90
+ <div style="padding: 15px; font-family: 'Consolas', 'Monaco', monospace; font-size: 13px; line-height: 1.6; color: #222; height: auto; overflow-y: visible; white-space: pre-wrap;">{code_html}</div>
91
+ </div>
92
+ <div style="flex: 1.2; display: flex; flex-direction: column; gap: 15px; max-height: 800px; position: sticky; top: 0;">
93
+ <div style="border: 1px solid #f5c6cb; border-radius: 8px; background-color: #fff; overflow: hidden; display: flex; flex-direction: column; flex: 1;">
94
+ <div style="background-color: #f8d7da; padding: 10px; text-align: center; color: #721c24; font-weight: bold; font-size: 13px; border-bottom: 1px solid #f5c6cb;">👤 Human Signals (Negative)</div>
95
+ <div style="padding: 0; overflow-y: auto; flex: 1; max-height: 350px;">
96
+ <table style="width:100%; font-size:12px; border-collapse: collapse; margin: 0;"><tbody>{human_table}</tbody></table>
97
+ </div>
98
+ </div>
99
+ <div style="border: 1px solid #c3e6cb; border-radius: 8px; background-color: #fff; overflow: hidden; display: flex; flex-direction: column; flex: 1;">
100
+ <div style="background-color: #d4edda; padding: 10px; text-align: center; color: #155724; font-weight: bold; font-size: 13px; border-bottom: 1px solid #c3e6cb;">🤖 AI Signals (Positive)</div>
101
+ <div style="padding: 0; overflow-y: auto; flex: 1; max-height: 350px;">
102
+ <table style="width:100%; font-size:12px; border-collapse: collapse; margin: 0;"><tbody>{ai_table}</tbody></table>
103
+ </div>
104
+ </div>
105
+ </div>
106
+ </div>
107
+ </body>
108
+ </html>
109
+ """
requirements.txt ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ torch>=2.0.0
2
+ transformers>=4.36.0
3
+ lightgbm>=4.0.0
4
+ captum>=0.7.0
5
+ scikit-learn>=1.3.0
6
+ numpy>=1.24.0
7
+ pandas>=2.0.0
8
+ lizard>=1.17.10
9
+ joblib>=1.3.0
10
+ gradio>=4.0.0
11
+ huggingface_hub