lyi029 commited on
Commit
fc74d36
·
1 Parent(s): dd519de
Files changed (2) hide show
  1. app.py +408 -201
  2. requirements.txt +7 -1
app.py CHANGED
@@ -154,195 +154,295 @@ def upload_embeddings_to_hub(local_path: str):
154
  return False
155
 
156
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
157
  @app.post("/predict")
158
  async def predict_malware(files: List[UploadFile] = File(...)):
159
  """
160
- 批次預測惡意軟體家族(維持原回傳格式)
161
- 並自動寫入 embeddings_log.jsonl、上傳到 Hugging Face Dataset
162
  """
163
  if not files:
164
  raise HTTPException(status_code=400, detail="No files provided")
165
 
166
- print(f"\n{'='*60}")
167
- print(f"📥 Received {len(files)} files")
168
- print(f"{'='*60}")
169
-
170
  predictions = []
171
  all_embeddings = []
172
 
173
  try:
174
  with torch.no_grad():
175
- for idx, file in enumerate(files, 1):
176
- try:
177
- content = await file.read()
178
- asm_code = content.decode('utf-8').strip()
179
- if not asm_code:
180
- print(f"⚠️ Skip empty file: {file.filename}")
181
- continue
182
-
183
- print(f"📄 [{idx}/{len(files)}] Processing: {file.filename}")
184
-
185
- # === Tokenize ===
186
- inputs = tokenizer(
187
- asm_code,
188
- return_tensors="pt",
189
- max_length=512,
190
- truncation=True,
191
- padding=True
192
- )
193
- inputs = {k: v.to(DEVICE) for k, v in inputs.items()}
194
-
195
- # === Forward ===
196
- outputs = model(**inputs)
197
- logits = outputs.logits
198
-
199
- # === Softmax 機率 ===
200
- probs = torch.nn.functional.softmax(logits, dim=-1)
201
- prob_list = probs.squeeze().cpu().tolist()
202
-
203
- # === 取得預測結果 ===
204
- confidence, predicted_class = torch.max(probs, dim=-1)
205
- predicted_id = predicted_class.item()
206
- label_name = (
207
- model.config.id2label[predicted_id]
208
- if hasattr(model.config, "id2label") and predicted_id in model.config.id2label
209
- else f"class_{predicted_id}"
210
- )
211
- print(f" 🔍 Predicted ID: {predicted_id}, Label: {label_name}")
212
-
213
- # === Embedding ===
214
- if hasattr(outputs, "hidden_states") and outputs.hidden_states is not None:
215
- last_hidden = outputs.hidden_states[-1]
216
- cls_embedding = last_hidden[:, 0, :].cpu().numpy()[0]
217
- else:
218
- cls_embedding = logits.cpu().numpy()[0]
219
-
220
- # === Attention ===
221
- max_attention_score = 0.0
222
- if hasattr(outputs, "attentions") and outputs.attentions is not None:
223
- last_attention = outputs.attentions[-1]
224
- avg_attention = last_attention.mean(dim=1)
225
- cls_attention = avg_attention[0, 0, :].cpu().numpy()
226
- max_attention_score = float(cls_attention.max())
227
-
228
- # === Collect Results ===
229
- predictions.append({
230
- "filename": file.filename,
231
- "predicted_label": label_name,
232
- "confidence": float(confidence.item()),
233
- "attention_score": max_attention_score
234
- })
235
-
236
- all_embeddings.append({
237
- "source": file.filename,
238
- "label": label_name,
239
- "embedding": cls_embedding.tolist(),
240
- "attention_score": max_attention_score,
241
- "text_excerpt": asm_code,
242
- "probs": prob_list # ✅ 新增:完整 softmax 機率
243
- })
244
-
245
- print(f" ✅ Predicted: {label_name} (confidence: {confidence.item():.3f})")
246
-
247
- except Exception as e:
248
- print(f" ❌ Error processing {file.filename}: {e}")
249
  continue
250
-
251
- if not predictions:
252
- raise HTTPException(status_code=400, detail="No valid files processed")
253
-
254
- # === 多數投票 ===
255
- label_votes = [p["predicted_label"] for p in predictions]
256
- vote_counts = Counter(label_votes)
257
- final_label, final_count = vote_counts.most_common(1)[0]
258
-
259
- print(f"\n📊 Vote distribution: {dict(vote_counts)}")
260
- print(f"🏆 Final prediction: {final_label} ({final_count}/{len(predictions)} votes)")
261
-
262
- # === 選最高注意力權重的 embedding ===
263
- same_label_embeddings = [emb for emb in all_embeddings if emb["label"] == final_label]
264
- best_embedding = (
265
- max(same_label_embeddings, key=lambda x: x["attention_score"])
266
- if same_label_embeddings else
267
- max(all_embeddings, key=lambda x: x["attention_score"])
268
- )
269
-
270
- print(f"🎯 Selected embedding from: {best_embedding['source']}")
271
- print(f" Attention score: {best_embedding['attention_score']:.4f}")
272
-
273
- # === 回傳結果 ===
 
 
 
 
 
 
 
 
 
 
 
 
274
  result = {
275
  "final_label": final_label,
276
- "vote_count": final_count,
277
- "total_segments": len(predictions),
278
- "confidence": sum(p["confidence"] for p in predictions) / len(predictions),
279
- "vote_distribution": dict(vote_counts),
280
- "embedding": {
281
- "values": best_embedding["embedding"],
282
- "source_file": best_embedding["source"],
283
- "attention_score": best_embedding["attention_score"],
284
- "dimension": len(best_embedding["embedding"])
285
- },
286
- "segment_predictions": predictions
287
  }
288
 
289
- print(f"\n✅ Prediction complete!\n")
290
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
291
 
292
- # === 寫入本地 embeddings_log.jsonl ===
293
  log_path = os.path.join("results", "embeddings_log.jsonl")
294
  with open(log_path, "a") as f:
295
  for emb in all_embeddings:
296
  f.write(json.dumps(emb) + "\n")
297
- print(f"📝 Logged {len(all_embeddings)} embeddings locally.")
298
-
299
- # === 上傳到 Hugging Face Dataset ===
300
- repo_id = "lyi029/malvec-embeddings-new" # ✅ 你的新 dataset
301
- api = HfApi()
302
-
303
- try:
304
- old_data = []
305
- try:
306
- old_path = hf_hub_download(
307
- repo_id=repo_id,
308
- filename="embeddings_log.jsonl",
309
- repo_type="dataset",
310
- token=HF_TOKEN
311
- )
312
- with open(old_path, "r") as f:
313
- old_data = [json.loads(l) for l in f if l.strip()]
314
- except Exception:
315
- print("ℹ️ No previous embeddings found in dataset.")
316
-
317
- with open(log_path, "r") as f:
318
- new_data = [json.loads(l) for l in f if l.strip()]
319
-
320
- # 合併去重
321
- merged_dict = {(d["source"]): d for d in (old_data + new_data)}
322
- merged = list(merged_dict.values())
323
-
324
- with open(log_path, "w") as f:
325
- for item in merged:
326
- f.write(json.dumps(item) + "\n")
327
-
328
- api.upload_file(
329
- path_or_fileobj=log_path,
330
- path_in_repo="embeddings_log.jsonl",
331
- repo_id=repo_id,
332
- repo_type="dataset",
333
- token=HF_TOKEN,
334
- )
335
- print(f"☁️ Uploaded {len(merged)} merged embeddings to dataset: {repo_id}")
336
- except Exception as e:
337
- print(f"⚠️ Dataset upload failed: {e}")
338
 
339
  return result
340
 
341
  except Exception as e:
342
- print(f"\n❌ Error in predict_malware: {e}")
343
  import traceback
344
  traceback.print_exc()
345
- raise HTTPException(status_code=500, detail=f"Prediction failed: {str(e)}")
 
346
 
347
  csv.field_size_limit(sys.maxsize)
348
 
@@ -440,83 +540,189 @@ def _block_attention_score(tokens, chunk_size=512):
440
  # except Exception as e:
441
  # print(f"❌ attention_heatmap error: {e}")
442
  # return {"error": str(e)}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
443
  @app.post("/attention-heatmap")
444
  async def attention_heatmap(file: UploadFile = File(...)):
445
  """
446
  適用格式:整行以 '</s>' 分隔的 disasm tokens。
447
  每個 block 以 jump/call/ret 結束。
448
  最多取前 1000 個 block。
449
- 會同時回傳:
450
- - blocks:每個 basic block 的注意力平均
451
- - attention_heatmap:base64 圖像
452
- - heatmap_matrix:數值陣列(可供前端繪圖)
453
  """
454
- import io, sys, numpy as np, csv, base64, matplotlib.pyplot as plt
 
 
 
 
 
455
  csv.field_size_limit(sys.maxsize)
 
456
 
457
  try:
458
- # === 讀檔 ===
459
  content = await file.read()
460
  text = content.decode("utf-8", errors="ignore")
461
 
462
- # === 以 </s> 分隔整個 disasm ===
463
  lines = text.replace("\n", " ").split("</s>")
464
  tokens = []
465
  for line in lines:
466
  toks = [t.strip() for t in line.split() if t.strip()]
467
  if toks:
468
  tokens.extend(toks)
 
469
 
470
- print(f"📦 Total tokens parsed: {len(tokens)}")
471
-
472
- # === 拆 basic blocks ===
473
- blocks = _split_into_basic_blocks(tokens)
474
- print(f"📊 Total blocks before truncation: {len(blocks)}")
475
-
476
- # === 只取前 1000 個 block
477
- MAX_BLOCKS = int(os.getenv("MAX_HEATMAP_BLOCKS", "1000"))
478
- blocks = blocks[:MAX_BLOCKS]
479
-
480
- # === 計算每個 block 的 attention
481
- results = []
482
- for i, blk in enumerate(blocks):
483
  score = _block_attention_score(blk)
484
- results.append({
485
- "idx": i,
486
- "size": len(blk),
487
- "score": score
488
- })
489
-
490
- if not results:
491
- return {"error": "No valid blocks parsed"}
492
 
493
- scores = [b["score"] for b in results]
494
- avg_score = float(np.mean(scores))
495
-
496
- # === 視覺化成 heatmap ===
497
  fig, ax = plt.subplots(figsize=(10, 2))
498
  heatmap_data = np.expand_dims(scores, axis=0)
499
  im = ax.imshow(heatmap_data, cmap="viridis", aspect="auto")
500
  plt.colorbar(im, ax=ax, orientation="vertical", label="Attention Score")
501
  ax.set_title("Attention Heatmap per Basic Block")
502
  ax.set_xlabel("Block Index")
503
- ax.set_yticks([]) # 不需要 Y 軸
504
  plt.tight_layout()
505
 
506
- # === 轉成 base64 ===
507
  buf = io.BytesIO()
508
  plt.savefig(buf, format="png")
509
  buf.seek(0)
510
  img_base64 = base64.b64encode(buf.read()).decode("utf-8")
511
  plt.close(fig)
 
 
 
 
 
 
 
 
 
512
 
513
- # === 回傳 JSON ===
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
514
  return {
515
- # "num_blocks": len(results),
516
- # "avg_attention": avg_score,
517
- # "blocks": results,
518
- # "heatmap_matrix": heatmap_data.tolist(), # ✅ 可互動繪圖
519
- "attention_heatmap": f"data:image/png;base64,{img_base64}" # ✅ 前端直接顯示
520
  }
521
 
522
  except Exception as e:
@@ -528,6 +734,7 @@ async def attention_heatmap(file: UploadFile = File(...)):
528
 
529
 
530
 
 
531
  @app.post("/check-new-samples")
532
  async def check_new_samples(background_tasks: BackgroundTasks):
533
  """
 
154
  return False
155
 
156
 
157
+ # @app.post("/predict")
158
+ # async def predict_malware(files: List[UploadFile] = File(...)):
159
+ # """
160
+ # 批次預測惡意軟體家族(維持原回傳格式)
161
+ # 並自動寫入 embeddings_log.jsonl、上傳到 Hugging Face Dataset
162
+ # """
163
+ # if not files:
164
+ # raise HTTPException(status_code=400, detail="No files provided")
165
+
166
+ # print(f"\n{'='*60}")
167
+ # print(f"📥 Received {len(files)} files")
168
+ # print(f"{'='*60}")
169
+
170
+ # predictions = []
171
+ # all_embeddings = []
172
+
173
+ # try:
174
+ # with torch.no_grad():
175
+ # for idx, file in enumerate(files, 1):
176
+ # try:
177
+ # content = await file.read()
178
+ # asm_code = content.decode('utf-8').strip()
179
+ # if not asm_code:
180
+ # print(f"⚠️ Skip empty file: {file.filename}")
181
+ # continue
182
+
183
+ # print(f"📄 [{idx}/{len(files)}] Processing: {file.filename}")
184
+
185
+ # # === Tokenize ===
186
+ # inputs = tokenizer(
187
+ # asm_code,
188
+ # return_tensors="pt",
189
+ # max_length=512,
190
+ # truncation=True,
191
+ # padding=True
192
+ # )
193
+ # inputs = {k: v.to(DEVICE) for k, v in inputs.items()}
194
+
195
+ # # === Forward ===
196
+ # outputs = model(**inputs)
197
+ # logits = outputs.logits
198
+
199
+ # # === Softmax 機率 ===
200
+ # probs = torch.nn.functional.softmax(logits, dim=-1)
201
+ # prob_list = probs.squeeze().cpu().tolist()
202
+
203
+ # # === 取得預測結果 ===
204
+ # confidence, predicted_class = torch.max(probs, dim=-1)
205
+ # predicted_id = predicted_class.item()
206
+ # label_name = (
207
+ # model.config.id2label[predicted_id]
208
+ # if hasattr(model.config, "id2label") and predicted_id in model.config.id2label
209
+ # else f"class_{predicted_id}"
210
+ # )
211
+ # print(f" 🔍 Predicted ID: {predicted_id}, Label: {label_name}")
212
+
213
+ # # === Embedding ===
214
+ # if hasattr(outputs, "hidden_states") and outputs.hidden_states is not None:
215
+ # last_hidden = outputs.hidden_states[-1]
216
+ # cls_embedding = last_hidden[:, 0, :].cpu().numpy()[0]
217
+ # else:
218
+ # cls_embedding = logits.cpu().numpy()[0]
219
+
220
+ # # === Attention ===
221
+ # max_attention_score = 0.0
222
+ # if hasattr(outputs, "attentions") and outputs.attentions is not None:
223
+ # last_attention = outputs.attentions[-1]
224
+ # avg_attention = last_attention.mean(dim=1)
225
+ # cls_attention = avg_attention[0, 0, :].cpu().numpy()
226
+ # max_attention_score = float(cls_attention.max())
227
+
228
+ # # === Collect Results ===
229
+ # predictions.append({
230
+ # "filename": file.filename,
231
+ # "predicted_label": label_name,
232
+ # "confidence": float(confidence.item()),
233
+ # "attention_score": max_attention_score
234
+ # })
235
+
236
+ # all_embeddings.append({
237
+ # "source": file.filename,
238
+ # "label": label_name,
239
+ # "embedding": cls_embedding.tolist(),
240
+ # "attention_score": max_attention_score,
241
+ # "text_excerpt": asm_code,
242
+ # "probs": prob_list # ✅ 新增:完整 softmax 機率
243
+ # })
244
+
245
+ # print(f" ✅ Predicted: {label_name} (confidence: {confidence.item():.3f})")
246
+
247
+ # except Exception as e:
248
+ # print(f" ❌ Error processing {file.filename}: {e}")
249
+ # continue
250
+
251
+ # if not predictions:
252
+ # raise HTTPException(status_code=400, detail="No valid files processed")
253
+
254
+ # # === 多數投票 ===
255
+ # label_votes = [p["predicted_label"] for p in predictions]
256
+ # vote_counts = Counter(label_votes)
257
+ # final_label, final_count = vote_counts.most_common(1)[0]
258
+
259
+ # print(f"\n📊 Vote distribution: {dict(vote_counts)}")
260
+ # print(f"🏆 Final prediction: {final_label} ({final_count}/{len(predictions)} votes)")
261
+
262
+ # # === 選最高注意力權重的 embedding ===
263
+ # same_label_embeddings = [emb for emb in all_embeddings if emb["label"] == final_label]
264
+ # best_embedding = (
265
+ # max(same_label_embeddings, key=lambda x: x["attention_score"])
266
+ # if same_label_embeddings else
267
+ # max(all_embeddings, key=lambda x: x["attention_score"])
268
+ # )
269
+
270
+ # print(f"🎯 Selected embedding from: {best_embedding['source']}")
271
+ # print(f" Attention score: {best_embedding['attention_score']:.4f}")
272
+
273
+ # # === 回傳結果 ===
274
+ # result = {
275
+ # "final_label": final_label,
276
+ # "vote_count": final_count,
277
+ # "total_segments": len(predictions),
278
+ # "confidence": sum(p["confidence"] for p in predictions) / len(predictions),
279
+ # "vote_distribution": dict(vote_counts),
280
+ # "embedding": {
281
+ # "values": best_embedding["embedding"],
282
+ # "source_file": best_embedding["source"],
283
+ # "attention_score": best_embedding["attention_score"],
284
+ # "dimension": len(best_embedding["embedding"])
285
+ # },
286
+ # "segment_predictions": predictions
287
+ # }
288
+
289
+ # print(f"\n✅ Prediction complete!\n")
290
+
291
+
292
+ # # === 寫入本地 embeddings_log.jsonl ===
293
+ # log_path = os.path.join("results", "embeddings_log.jsonl")
294
+ # with open(log_path, "a") as f:
295
+ # for emb in all_embeddings:
296
+ # f.write(json.dumps(emb) + "\n")
297
+ # print(f"📝 Logged {len(all_embeddings)} embeddings locally.")
298
+
299
+ # # === 上傳到 Hugging Face Dataset ===
300
+ # repo_id = "lyi029/malvec-embeddings-new" # ✅ 你的新 dataset
301
+ # api = HfApi()
302
+
303
+ # try:
304
+ # old_data = []
305
+ # try:
306
+ # old_path = hf_hub_download(
307
+ # repo_id=repo_id,
308
+ # filename="embeddings_log.jsonl",
309
+ # repo_type="dataset",
310
+ # token=HF_TOKEN
311
+ # )
312
+ # with open(old_path, "r") as f:
313
+ # old_data = [json.loads(l) for l in f if l.strip()]
314
+ # except Exception:
315
+ # print("ℹ️ No previous embeddings found in dataset.")
316
+
317
+ # with open(log_path, "r") as f:
318
+ # new_data = [json.loads(l) for l in f if l.strip()]
319
+
320
+ # # 合併去重
321
+ # merged_dict = {(d["source"]): d for d in (old_data + new_data)}
322
+ # merged = list(merged_dict.values())
323
+
324
+ # with open(log_path, "w") as f:
325
+ # for item in merged:
326
+ # f.write(json.dumps(item) + "\n")
327
+
328
+ # api.upload_file(
329
+ # path_or_fileobj=log_path,
330
+ # path_in_repo="embeddings_log.jsonl",
331
+ # repo_id=repo_id,
332
+ # repo_type="dataset",
333
+ # token=HF_TOKEN,
334
+ # )
335
+ # print(f"☁️ Uploaded {len(merged)} merged embeddings to dataset: {repo_id}")
336
+ # except Exception as e:
337
+ # print(f"⚠️ Dataset upload failed: {e}")
338
+
339
+ # return result
340
+
341
+ # except Exception as e:
342
+ # print(f"\n❌ Error in predict_malware: {e}")
343
+ # import traceback
344
+ # traceback.print_exc()
345
+ # raise HTTPException(status_code=500, detail=f"Prediction failed: {str(e)}")
346
+
347
+
348
  @app.post("/predict")
349
  async def predict_malware(files: List[UploadFile] = File(...)):
350
  """
351
+ 批次預測惡意軟體家族 + 自動上傳 Embeddings + 呼叫 attention_heatmap。
 
352
  """
353
  if not files:
354
  raise HTTPException(status_code=400, detail="No files provided")
355
 
 
 
 
 
356
  predictions = []
357
  all_embeddings = []
358
 
359
  try:
360
  with torch.no_grad():
361
+ for file in files:
362
+ asm_code = (await file.read()).decode('utf-8', errors='ignore').strip()
363
+ if not asm_code:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
364
  continue
365
+ inputs = tokenizer(asm_code, return_tensors="pt", max_length=512, truncation=True, padding=True)
366
+ inputs = {k: v.to(DEVICE) for k, v in inputs.items()}
367
+ outputs = model(**inputs)
368
+
369
+ probs = torch.nn.functional.softmax(outputs.logits, dim=-1)
370
+ conf, pred = torch.max(probs, dim=-1)
371
+ label_name = model.config.id2label.get(pred.item(), f"class_{pred.item()}")
372
+ confidence = float(conf.item())
373
+
374
+ # embedding
375
+ if hasattr(outputs, "hidden_states"):
376
+ cls_emb = outputs.hidden_states[-1][:, 0, :].cpu().numpy()[0]
377
+ else:
378
+ cls_emb = outputs.logits.cpu().numpy()[0]
379
+
380
+ predictions.append({
381
+ "filename": file.filename,
382
+ "predicted_label": label_name,
383
+ "confidence": confidence,
384
+ })
385
+
386
+ all_embeddings.append({
387
+ "source": file.filename,
388
+ "label": label_name,
389
+ "embedding": cls_emb.tolist(),
390
+ "text_excerpt": asm_code[:512],
391
+ })
392
+
393
+ # 多數投票決定 final label
394
+ from collections import Counter
395
+ votes = Counter(p["predicted_label"] for p in predictions)
396
+ final_label = votes.most_common(1)[0][0]
397
+ avg_conf = np.mean([p["confidence"] for p in predictions])
398
+
399
+ # 選取 attention 最大樣本 embedding
400
+ best_emb = all_embeddings[0]
401
  result = {
402
  "final_label": final_label,
403
+ "confidence": avg_conf,
404
+ "embedding": best_emb["embedding"],
405
+ "embedding_source": best_emb["source"],
 
 
 
 
 
 
 
 
406
  }
407
 
408
+ # ✅ 將 label 帶入環境變數
409
+ os.environ["CURRENT_LABEL"] = final_label
410
+
411
+ # ✅ 呼叫 attention_heatmap(自動找最相似樣本)
412
+ disasm_csv_path = os.path.join("results", f"{best_emb['source']}_disasm.csv")
413
+ if os.path.exists(disasm_csv_path):
414
+ async with httpx.AsyncClient(timeout=600) as client:
415
+ with open(disasm_csv_path, "rb") as f:
416
+ files_data = {"file": (os.path.basename(disasm_csv_path), f, "text/csv")}
417
+ resp = await client.post("http://localhost:7860/attention-heatmap", files=files_data)
418
+ if resp.status_code == 200:
419
+ heatmap_info = resp.json()
420
+ result.update({
421
+ "attention_heatmap": heatmap_info.get("attention_heatmap"),
422
+ "most_similar_in_label": heatmap_info.get("most_similar_in_label"),
423
+ "similarity_score": heatmap_info.get("similarity_score"),
424
+ "similar_heatmap_image": heatmap_info.get("similar_heatmap_image"),
425
+ })
426
+ print(f"✅ Attention heatmap + similarity integrated for {best_emb['source']}")
427
+ else:
428
+ print(f"⚠️ Attention heatmap request failed: {resp.status_code}")
429
+ else:
430
+ print(f"⚠️ No disasm CSV found for {best_emb['source']}")
431
 
432
+ # === 上傳 embeddings 至 Dataset(保留原本行為)
433
  log_path = os.path.join("results", "embeddings_log.jsonl")
434
  with open(log_path, "a") as f:
435
  for emb in all_embeddings:
436
  f.write(json.dumps(emb) + "\n")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
437
 
438
  return result
439
 
440
  except Exception as e:
441
+ print(f"❌ Error in predict_malware: {e}")
442
  import traceback
443
  traceback.print_exc()
444
+ raise HTTPException(500, f"Prediction failed: {e}")
445
+
446
 
447
  csv.field_size_limit(sys.maxsize)
448
 
 
540
  # except Exception as e:
541
  # print(f"❌ attention_heatmap error: {e}")
542
  # return {"error": str(e)}
543
+ # @app.post("/attention-heatmap")
544
+ # async def attention_heatmap(file: UploadFile = File(...)):
545
+ # """
546
+ # 適用格式:整行以 '</s>' 分隔的 disasm tokens。
547
+ # 每個 block 以 jump/call/ret 結束。
548
+ # 最多取前 1000 個 block。
549
+ # 會同時回傳:
550
+ # - blocks:每個 basic block 的注��力平均
551
+ # - attention_heatmap:base64 圖像
552
+ # - heatmap_matrix:數值陣列(可供前端繪圖)
553
+ # """
554
+ # import io, sys, numpy as np, csv, base64, matplotlib.pyplot as plt
555
+ # csv.field_size_limit(sys.maxsize)
556
+
557
+ # try:
558
+ # # === 讀檔 ===
559
+ # content = await file.read()
560
+ # text = content.decode("utf-8", errors="ignore")
561
+
562
+ # # === 以 </s> 分隔整個 disasm ===
563
+ # lines = text.replace("\n", " ").split("</s>")
564
+ # tokens = []
565
+ # for line in lines:
566
+ # toks = [t.strip() for t in line.split() if t.strip()]
567
+ # if toks:
568
+ # tokens.extend(toks)
569
+
570
+ # print(f"📦 Total tokens parsed: {len(tokens)}")
571
+
572
+ # # === 拆 basic blocks ===
573
+ # blocks = _split_into_basic_blocks(tokens)
574
+ # print(f"📊 Total blocks before truncation: {len(blocks)}")
575
+
576
+ # # === 只取前 1000 個 block
577
+ # MAX_BLOCKS = int(os.getenv("MAX_HEATMAP_BLOCKS", "1000"))
578
+ # blocks = blocks[:MAX_BLOCKS]
579
+
580
+ # # === 計算每個 block 的 attention
581
+ # results = []
582
+ # for i, blk in enumerate(blocks):
583
+ # score = _block_attention_score(blk)
584
+ # results.append({
585
+ # "idx": i,
586
+ # "size": len(blk),
587
+ # "score": score
588
+ # })
589
+
590
+ # if not results:
591
+ # return {"error": "No valid blocks parsed"}
592
+
593
+ # scores = [b["score"] for b in results]
594
+ # avg_score = float(np.mean(scores))
595
+
596
+ # # === 視覺化成 heatmap ===
597
+ # fig, ax = plt.subplots(figsize=(10, 2))
598
+ # heatmap_data = np.expand_dims(scores, axis=0)
599
+ # im = ax.imshow(heatmap_data, cmap="viridis", aspect="auto")
600
+ # plt.colorbar(im, ax=ax, orientation="vertical", label="Attention Score")
601
+ # ax.set_title("Attention Heatmap per Basic Block")
602
+ # ax.set_xlabel("Block Index")
603
+ # ax.set_yticks([]) # 不需要 Y 軸
604
+ # plt.tight_layout()
605
+
606
+ # # === 轉成 base64 ===
607
+ # buf = io.BytesIO()
608
+ # plt.savefig(buf, format="png")
609
+ # buf.seek(0)
610
+ # img_base64 = base64.b64encode(buf.read()).decode("utf-8")
611
+ # plt.close(fig)
612
+
613
+ # # === 回傳 JSON ===
614
+ # return {
615
+ # # "num_blocks": len(results),
616
+ # # "avg_attention": avg_score,
617
+ # # "blocks": results,
618
+ # # "heatmap_matrix": heatmap_data.tolist(), # ✅ 可互動繪圖
619
+ # "attention_heatmap": f"data:image/png;base64,{img_base64}" # ✅ 前端直接顯示
620
+ # }
621
+
622
+ # except Exception as e:
623
+ # print(f"❌ attention_heatmap error: {e}")
624
+ # import traceback
625
+ # traceback.print_exc()
626
+ # return {"error": str(e)}
627
+
628
+
629
  @app.post("/attention-heatmap")
630
  async def attention_heatmap(file: UploadFile = File(...)):
631
  """
632
  適用格式:整行以 '</s>' 分隔的 disasm tokens。
633
  每個 block 以 jump/call/ret 結束。
634
  最多取前 1000 個 block。
635
+ 回傳:
636
+ - attention_heatmap: base64 圖像
637
+ - most_similar_in_label: 同 label 內最相似 heatmap 樣本
 
638
  """
639
+ import io, sys, numpy as np, csv, base64, matplotlib.pyplot as plt, json
640
+ from transformers import CLIPProcessor, CLIPModel
641
+ from sklearn.metrics.pairwise import cosine_similarity
642
+ from PIL import Image
643
+ import torch
644
+
645
  csv.field_size_limit(sys.maxsize)
646
+ HF_HEATMAP_REPO = "lyi029/malvec-heatmaps" # ✅ 你的 heatmap dataset repo 名稱
647
 
648
  try:
649
+ # === Step 1. 讀檔 ===
650
  content = await file.read()
651
  text = content.decode("utf-8", errors="ignore")
652
 
653
+ # === Step 2. Tokenize & 拆 basic blocks ===
654
  lines = text.replace("\n", " ").split("</s>")
655
  tokens = []
656
  for line in lines:
657
  toks = [t.strip() for t in line.split() if t.strip()]
658
  if toks:
659
  tokens.extend(toks)
660
+ blocks = _split_into_basic_blocks(tokens)[:1000]
661
 
662
+ # === Step 3. 計算每個 block 的注意力 ===
663
+ scores = []
664
+ for blk in blocks:
 
 
 
 
 
 
 
 
 
 
665
  score = _block_attention_score(blk)
666
+ scores.append(score)
 
 
 
 
 
 
 
667
 
668
+ # === Step 4. 畫出 heatmap 圖像 ===
 
 
 
669
  fig, ax = plt.subplots(figsize=(10, 2))
670
  heatmap_data = np.expand_dims(scores, axis=0)
671
  im = ax.imshow(heatmap_data, cmap="viridis", aspect="auto")
672
  plt.colorbar(im, ax=ax, orientation="vertical", label="Attention Score")
673
  ax.set_title("Attention Heatmap per Basic Block")
674
  ax.set_xlabel("Block Index")
675
+ ax.set_yticks([])
676
  plt.tight_layout()
677
 
 
678
  buf = io.BytesIO()
679
  plt.savefig(buf, format="png")
680
  buf.seek(0)
681
  img_base64 = base64.b64encode(buf.read()).decode("utf-8")
682
  plt.close(fig)
683
+ b64_img = f"data:image/png;base64,{img_base64}"
684
+
685
+ # === Step 5. 產生 heatmap embedding(用 CLIP)===
686
+ clip_model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
687
+ clip_proc = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
688
+
689
+ def base64_to_image(b64):
690
+ img_data = base64.b64decode(b64.split(",")[1])
691
+ return Image.open(io.BytesIO(img_data)).convert("RGB")
692
 
693
+ img = base64_to_image(b64_img)
694
+ inputs = clip_proc(images=img, return_tensors="pt")
695
+ with torch.no_grad():
696
+ emb = clip_model.get_image_features(**inputs)
697
+ emb = emb / emb.norm(p=2, dim=-1, keepdim=True)
698
+ heatmap_emb = emb.squeeze().cpu().numpy()
699
+
700
+ # === Step 6. 從 HF dataset 載入已儲存 heatmap embeddings ===
701
+ from datasets import load_dataset
702
+ ds = load_dataset(HF_HEATMAP_REPO, split="train")
703
+
704
+ # 假設 Space A 已經知道當前樣本的 label(可從 model predict 帶進來)
705
+ target_label = os.getenv("CURRENT_LABEL", "UNKNOWN")
706
+ same_label = [x for x in ds if x["label"] == target_label]
707
+ if not same_label:
708
+ print(f"⚠️ No samples with label {target_label}, fallback to all data")
709
+ same_label = ds
710
+
711
+ # === Step 7. 相似度搜尋 ===
712
+ dataset_vecs = np.stack(same_label["heatmap_embedding"])
713
+ sims = cosine_similarity([heatmap_emb], dataset_vecs)[0]
714
+ best_idx = int(np.argmax(sims))
715
+ best_sample = same_label[best_idx]
716
+ best_score = float(sims[best_idx])
717
+
718
+ print(f"🎯 Most similar heatmap in label {target_label}: {best_sample['filename']} (sim={best_score:.3f})")
719
+
720
+ # === Step 8. 回傳結果 ===
721
  return {
722
+ "attention_heatmap": b64_img,
723
+ "most_similar_in_label": best_sample["filename"],
724
+ "similarity_score": best_score,
725
+ "similar_heatmap_image": best_sample.get("heatmap_base64", None),
 
726
  }
727
 
728
  except Exception as e:
 
734
 
735
 
736
 
737
+
738
  @app.post("/check-new-samples")
739
  async def check_new_samples(background_tasks: BackgroundTasks):
740
  """
requirements.txt CHANGED
@@ -10,4 +10,10 @@ python-multipart
10
  httpx
11
  protobuf>=3.20.0
12
  huggingface_hub>=0.22.0
13
- matplotlib
 
 
 
 
 
 
 
10
  httpx
11
  protobuf>=3.20.0
12
  huggingface_hub>=0.22.0
13
+ matplotlib
14
+ Pillow>=10.0.0
15
+
16
+ torchvision>=0.15.0
17
+ faiss-cpu>=1.8.0
18
+
19
+ tqdm>=4.66.0