lyi029 commited on
Commit
774d4f3
·
1 Parent(s): fc74d36
Files changed (1) hide show
  1. app.py +201 -408
app.py CHANGED
@@ -154,295 +154,195 @@ def upload_embeddings_to_hub(local_path: str):
154
  return False
155
 
156
 
157
- # @app.post("/predict")
158
- # async def predict_malware(files: List[UploadFile] = File(...)):
159
- # """
160
- # 批次預測惡意軟體家族(維持原回傳格式)
161
- # 並自動寫入 embeddings_log.jsonl、上傳到 Hugging Face Dataset
162
- # """
163
- # if not files:
164
- # raise HTTPException(status_code=400, detail="No files provided")
165
-
166
- # print(f"\n{'='*60}")
167
- # print(f"📥 Received {len(files)} files")
168
- # print(f"{'='*60}")
169
-
170
- # predictions = []
171
- # all_embeddings = []
172
-
173
- # try:
174
- # with torch.no_grad():
175
- # for idx, file in enumerate(files, 1):
176
- # try:
177
- # content = await file.read()
178
- # asm_code = content.decode('utf-8').strip()
179
- # if not asm_code:
180
- # print(f"⚠️ Skip empty file: {file.filename}")
181
- # continue
182
-
183
- # print(f"📄 [{idx}/{len(files)}] Processing: {file.filename}")
184
-
185
- # # === Tokenize ===
186
- # inputs = tokenizer(
187
- # asm_code,
188
- # return_tensors="pt",
189
- # max_length=512,
190
- # truncation=True,
191
- # padding=True
192
- # )
193
- # inputs = {k: v.to(DEVICE) for k, v in inputs.items()}
194
-
195
- # # === Forward ===
196
- # outputs = model(**inputs)
197
- # logits = outputs.logits
198
-
199
- # # === Softmax 機率 ===
200
- # probs = torch.nn.functional.softmax(logits, dim=-1)
201
- # prob_list = probs.squeeze().cpu().tolist()
202
-
203
- # # === 取得預測結果 ===
204
- # confidence, predicted_class = torch.max(probs, dim=-1)
205
- # predicted_id = predicted_class.item()
206
- # label_name = (
207
- # model.config.id2label[predicted_id]
208
- # if hasattr(model.config, "id2label") and predicted_id in model.config.id2label
209
- # else f"class_{predicted_id}"
210
- # )
211
- # print(f" 🔍 Predicted ID: {predicted_id}, Label: {label_name}")
212
-
213
- # # === Embedding ===
214
- # if hasattr(outputs, "hidden_states") and outputs.hidden_states is not None:
215
- # last_hidden = outputs.hidden_states[-1]
216
- # cls_embedding = last_hidden[:, 0, :].cpu().numpy()[0]
217
- # else:
218
- # cls_embedding = logits.cpu().numpy()[0]
219
-
220
- # # === Attention ===
221
- # max_attention_score = 0.0
222
- # if hasattr(outputs, "attentions") and outputs.attentions is not None:
223
- # last_attention = outputs.attentions[-1]
224
- # avg_attention = last_attention.mean(dim=1)
225
- # cls_attention = avg_attention[0, 0, :].cpu().numpy()
226
- # max_attention_score = float(cls_attention.max())
227
-
228
- # # === Collect Results ===
229
- # predictions.append({
230
- # "filename": file.filename,
231
- # "predicted_label": label_name,
232
- # "confidence": float(confidence.item()),
233
- # "attention_score": max_attention_score
234
- # })
235
-
236
- # all_embeddings.append({
237
- # "source": file.filename,
238
- # "label": label_name,
239
- # "embedding": cls_embedding.tolist(),
240
- # "attention_score": max_attention_score,
241
- # "text_excerpt": asm_code,
242
- # "probs": prob_list # ✅ 新增:完整 softmax 機率
243
- # })
244
-
245
- # print(f" ✅ Predicted: {label_name} (confidence: {confidence.item():.3f})")
246
-
247
- # except Exception as e:
248
- # print(f" ❌ Error processing {file.filename}: {e}")
249
- # continue
250
-
251
- # if not predictions:
252
- # raise HTTPException(status_code=400, detail="No valid files processed")
253
-
254
- # # === 多數投票 ===
255
- # label_votes = [p["predicted_label"] for p in predictions]
256
- # vote_counts = Counter(label_votes)
257
- # final_label, final_count = vote_counts.most_common(1)[0]
258
-
259
- # print(f"\n📊 Vote distribution: {dict(vote_counts)}")
260
- # print(f"🏆 Final prediction: {final_label} ({final_count}/{len(predictions)} votes)")
261
-
262
- # # === 選最高注意力權重的 embedding ===
263
- # same_label_embeddings = [emb for emb in all_embeddings if emb["label"] == final_label]
264
- # best_embedding = (
265
- # max(same_label_embeddings, key=lambda x: x["attention_score"])
266
- # if same_label_embeddings else
267
- # max(all_embeddings, key=lambda x: x["attention_score"])
268
- # )
269
-
270
- # print(f"🎯 Selected embedding from: {best_embedding['source']}")
271
- # print(f" Attention score: {best_embedding['attention_score']:.4f}")
272
-
273
- # # === 回傳結果 ===
274
- # result = {
275
- # "final_label": final_label,
276
- # "vote_count": final_count,
277
- # "total_segments": len(predictions),
278
- # "confidence": sum(p["confidence"] for p in predictions) / len(predictions),
279
- # "vote_distribution": dict(vote_counts),
280
- # "embedding": {
281
- # "values": best_embedding["embedding"],
282
- # "source_file": best_embedding["source"],
283
- # "attention_score": best_embedding["attention_score"],
284
- # "dimension": len(best_embedding["embedding"])
285
- # },
286
- # "segment_predictions": predictions
287
- # }
288
-
289
- # print(f"\n✅ Prediction complete!\n")
290
-
291
-
292
- # # === 寫入本地 embeddings_log.jsonl ===
293
- # log_path = os.path.join("results", "embeddings_log.jsonl")
294
- # with open(log_path, "a") as f:
295
- # for emb in all_embeddings:
296
- # f.write(json.dumps(emb) + "\n")
297
- # print(f"📝 Logged {len(all_embeddings)} embeddings locally.")
298
-
299
- # # === 上傳到 Hugging Face Dataset ===
300
- # repo_id = "lyi029/malvec-embeddings-new" # ✅ 你的新 dataset
301
- # api = HfApi()
302
-
303
- # try:
304
- # old_data = []
305
- # try:
306
- # old_path = hf_hub_download(
307
- # repo_id=repo_id,
308
- # filename="embeddings_log.jsonl",
309
- # repo_type="dataset",
310
- # token=HF_TOKEN
311
- # )
312
- # with open(old_path, "r") as f:
313
- # old_data = [json.loads(l) for l in f if l.strip()]
314
- # except Exception:
315
- # print("ℹ️ No previous embeddings found in dataset.")
316
-
317
- # with open(log_path, "r") as f:
318
- # new_data = [json.loads(l) for l in f if l.strip()]
319
-
320
- # # 合併去重
321
- # merged_dict = {(d["source"]): d for d in (old_data + new_data)}
322
- # merged = list(merged_dict.values())
323
-
324
- # with open(log_path, "w") as f:
325
- # for item in merged:
326
- # f.write(json.dumps(item) + "\n")
327
-
328
- # api.upload_file(
329
- # path_or_fileobj=log_path,
330
- # path_in_repo="embeddings_log.jsonl",
331
- # repo_id=repo_id,
332
- # repo_type="dataset",
333
- # token=HF_TOKEN,
334
- # )
335
- # print(f"☁️ Uploaded {len(merged)} merged embeddings to dataset: {repo_id}")
336
- # except Exception as e:
337
- # print(f"⚠️ Dataset upload failed: {e}")
338
-
339
- # return result
340
-
341
- # except Exception as e:
342
- # print(f"\n❌ Error in predict_malware: {e}")
343
- # import traceback
344
- # traceback.print_exc()
345
- # raise HTTPException(status_code=500, detail=f"Prediction failed: {str(e)}")
346
-
347
-
348
  @app.post("/predict")
349
  async def predict_malware(files: List[UploadFile] = File(...)):
350
  """
351
- 批次預測惡意軟體家族 + 自動上傳 Embeddings + 呼叫 attention_heatmap。
 
352
  """
353
  if not files:
354
  raise HTTPException(status_code=400, detail="No files provided")
355
 
 
 
 
 
356
  predictions = []
357
  all_embeddings = []
358
 
359
  try:
360
  with torch.no_grad():
361
- for file in files:
362
- asm_code = (await file.read()).decode('utf-8', errors='ignore').strip()
363
- if not asm_code:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
364
  continue
365
- inputs = tokenizer(asm_code, return_tensors="pt", max_length=512, truncation=True, padding=True)
366
- inputs = {k: v.to(DEVICE) for k, v in inputs.items()}
367
- outputs = model(**inputs)
368
-
369
- probs = torch.nn.functional.softmax(outputs.logits, dim=-1)
370
- conf, pred = torch.max(probs, dim=-1)
371
- label_name = model.config.id2label.get(pred.item(), f"class_{pred.item()}")
372
- confidence = float(conf.item())
373
-
374
- # embedding
375
- if hasattr(outputs, "hidden_states"):
376
- cls_emb = outputs.hidden_states[-1][:, 0, :].cpu().numpy()[0]
377
- else:
378
- cls_emb = outputs.logits.cpu().numpy()[0]
379
-
380
- predictions.append({
381
- "filename": file.filename,
382
- "predicted_label": label_name,
383
- "confidence": confidence,
384
- })
385
-
386
- all_embeddings.append({
387
- "source": file.filename,
388
- "label": label_name,
389
- "embedding": cls_emb.tolist(),
390
- "text_excerpt": asm_code[:512],
391
- })
392
-
393
- # 多數投票決定 final label
394
- from collections import Counter
395
- votes = Counter(p["predicted_label"] for p in predictions)
396
- final_label = votes.most_common(1)[0][0]
397
- avg_conf = np.mean([p["confidence"] for p in predictions])
398
-
399
- # 選取 attention 最大樣本 embedding
400
- best_emb = all_embeddings[0]
401
  result = {
402
  "final_label": final_label,
403
- "confidence": avg_conf,
404
- "embedding": best_emb["embedding"],
405
- "embedding_source": best_emb["source"],
 
 
 
 
 
 
 
 
406
  }
407
 
408
- # ✅ 將 label 帶入環境變數
409
- os.environ["CURRENT_LABEL"] = final_label
410
-
411
- # ✅ 呼叫 attention_heatmap(自動找最相似樣本)
412
- disasm_csv_path = os.path.join("results", f"{best_emb['source']}_disasm.csv")
413
- if os.path.exists(disasm_csv_path):
414
- async with httpx.AsyncClient(timeout=600) as client:
415
- with open(disasm_csv_path, "rb") as f:
416
- files_data = {"file": (os.path.basename(disasm_csv_path), f, "text/csv")}
417
- resp = await client.post("http://localhost:7860/attention-heatmap", files=files_data)
418
- if resp.status_code == 200:
419
- heatmap_info = resp.json()
420
- result.update({
421
- "attention_heatmap": heatmap_info.get("attention_heatmap"),
422
- "most_similar_in_label": heatmap_info.get("most_similar_in_label"),
423
- "similarity_score": heatmap_info.get("similarity_score"),
424
- "similar_heatmap_image": heatmap_info.get("similar_heatmap_image"),
425
- })
426
- print(f"✅ Attention heatmap + similarity integrated for {best_emb['source']}")
427
- else:
428
- print(f"⚠️ Attention heatmap request failed: {resp.status_code}")
429
- else:
430
- print(f"⚠️ No disasm CSV found for {best_emb['source']}")
431
 
432
- # === 上傳 embeddings 至 Dataset(保留原本行為)
 
433
  log_path = os.path.join("results", "embeddings_log.jsonl")
434
  with open(log_path, "a") as f:
435
  for emb in all_embeddings:
436
  f.write(json.dumps(emb) + "\n")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
437
 
438
  return result
439
 
440
  except Exception as e:
441
- print(f"❌ Error in predict_malware: {e}")
442
  import traceback
443
  traceback.print_exc()
444
- raise HTTPException(500, f"Prediction failed: {e}")
445
-
446
 
447
  csv.field_size_limit(sys.maxsize)
448
 
@@ -540,189 +440,83 @@ def _block_attention_score(tokens, chunk_size=512):
540
  # except Exception as e:
541
  # print(f"❌ attention_heatmap error: {e}")
542
  # return {"error": str(e)}
543
- # @app.post("/attention-heatmap")
544
- # async def attention_heatmap(file: UploadFile = File(...)):
545
- # """
546
- # 適用格式:整行以 '</s>' 分隔的 disasm tokens。
547
- # 每個 block 以 jump/call/ret 結束。
548
- # 最多取前 1000 個 block。
549
- # 會同時回傳:
550
- # - blocks:每個 basic block 的注��力平均
551
- # - attention_heatmap:base64 圖像
552
- # - heatmap_matrix:數值陣列(可供前端繪圖)
553
- # """
554
- # import io, sys, numpy as np, csv, base64, matplotlib.pyplot as plt
555
- # csv.field_size_limit(sys.maxsize)
556
-
557
- # try:
558
- # # === 讀檔 ===
559
- # content = await file.read()
560
- # text = content.decode("utf-8", errors="ignore")
561
-
562
- # # === 以 </s> 分隔整個 disasm ===
563
- # lines = text.replace("\n", " ").split("</s>")
564
- # tokens = []
565
- # for line in lines:
566
- # toks = [t.strip() for t in line.split() if t.strip()]
567
- # if toks:
568
- # tokens.extend(toks)
569
-
570
- # print(f"📦 Total tokens parsed: {len(tokens)}")
571
-
572
- # # === 拆 basic blocks ===
573
- # blocks = _split_into_basic_blocks(tokens)
574
- # print(f"📊 Total blocks before truncation: {len(blocks)}")
575
-
576
- # # === 只取前 1000 個 block
577
- # MAX_BLOCKS = int(os.getenv("MAX_HEATMAP_BLOCKS", "1000"))
578
- # blocks = blocks[:MAX_BLOCKS]
579
-
580
- # # === 計算每個 block 的 attention
581
- # results = []
582
- # for i, blk in enumerate(blocks):
583
- # score = _block_attention_score(blk)
584
- # results.append({
585
- # "idx": i,
586
- # "size": len(blk),
587
- # "score": score
588
- # })
589
-
590
- # if not results:
591
- # return {"error": "No valid blocks parsed"}
592
-
593
- # scores = [b["score"] for b in results]
594
- # avg_score = float(np.mean(scores))
595
-
596
- # # === 視覺化成 heatmap ===
597
- # fig, ax = plt.subplots(figsize=(10, 2))
598
- # heatmap_data = np.expand_dims(scores, axis=0)
599
- # im = ax.imshow(heatmap_data, cmap="viridis", aspect="auto")
600
- # plt.colorbar(im, ax=ax, orientation="vertical", label="Attention Score")
601
- # ax.set_title("Attention Heatmap per Basic Block")
602
- # ax.set_xlabel("Block Index")
603
- # ax.set_yticks([]) # 不需要 Y 軸
604
- # plt.tight_layout()
605
-
606
- # # === 轉成 base64 ===
607
- # buf = io.BytesIO()
608
- # plt.savefig(buf, format="png")
609
- # buf.seek(0)
610
- # img_base64 = base64.b64encode(buf.read()).decode("utf-8")
611
- # plt.close(fig)
612
-
613
- # # === 回傳 JSON ===
614
- # return {
615
- # # "num_blocks": len(results),
616
- # # "avg_attention": avg_score,
617
- # # "blocks": results,
618
- # # "heatmap_matrix": heatmap_data.tolist(), # ✅ 可互動繪圖
619
- # "attention_heatmap": f"data:image/png;base64,{img_base64}" # ✅ 前端直接顯示
620
- # }
621
-
622
- # except Exception as e:
623
- # print(f"❌ attention_heatmap error: {e}")
624
- # import traceback
625
- # traceback.print_exc()
626
- # return {"error": str(e)}
627
-
628
-
629
  @app.post("/attention-heatmap")
630
  async def attention_heatmap(file: UploadFile = File(...)):
631
  """
632
  適用格式:整行以 '</s>' 分隔的 disasm tokens。
633
  每個 block 以 jump/call/ret 結束。
634
  最多取前 1000 個 block。
635
- 回傳:
636
- - attention_heatmap: base64 圖像
637
- - most_similar_in_label: 同 label 內最相似 heatmap 樣本
 
638
  """
639
- import io, sys, numpy as np, csv, base64, matplotlib.pyplot as plt, json
640
- from transformers import CLIPProcessor, CLIPModel
641
- from sklearn.metrics.pairwise import cosine_similarity
642
- from PIL import Image
643
- import torch
644
-
645
  csv.field_size_limit(sys.maxsize)
646
- HF_HEATMAP_REPO = "lyi029/malvec-heatmaps" # ✅ 你的 heatmap dataset repo 名稱
647
 
648
  try:
649
- # === Step 1. 讀檔 ===
650
  content = await file.read()
651
  text = content.decode("utf-8", errors="ignore")
652
 
653
- # === Step 2. Tokenize & 拆 basic blocks ===
654
  lines = text.replace("\n", " ").split("</s>")
655
  tokens = []
656
  for line in lines:
657
  toks = [t.strip() for t in line.split() if t.strip()]
658
  if toks:
659
  tokens.extend(toks)
660
- blocks = _split_into_basic_blocks(tokens)[:1000]
661
 
662
- # === Step 3. 計算每個 block 的注意力 ===
663
- scores = []
664
- for blk in blocks:
 
 
 
 
 
 
 
 
 
 
665
  score = _block_attention_score(blk)
666
- scores.append(score)
 
 
 
 
 
 
 
667
 
668
- # === Step 4. 畫出 heatmap 圖像 ===
 
 
 
669
  fig, ax = plt.subplots(figsize=(10, 2))
670
  heatmap_data = np.expand_dims(scores, axis=0)
671
  im = ax.imshow(heatmap_data, cmap="viridis", aspect="auto")
672
  plt.colorbar(im, ax=ax, orientation="vertical", label="Attention Score")
673
  ax.set_title("Attention Heatmap per Basic Block")
674
  ax.set_xlabel("Block Index")
675
- ax.set_yticks([])
676
  plt.tight_layout()
677
 
 
678
  buf = io.BytesIO()
679
  plt.savefig(buf, format="png")
680
  buf.seek(0)
681
  img_base64 = base64.b64encode(buf.read()).decode("utf-8")
682
  plt.close(fig)
683
- b64_img = f"data:image/png;base64,{img_base64}"
684
-
685
- # === Step 5. 產生 heatmap embedding(用 CLIP)===
686
- clip_model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
687
- clip_proc = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
688
-
689
- def base64_to_image(b64):
690
- img_data = base64.b64decode(b64.split(",")[1])
691
- return Image.open(io.BytesIO(img_data)).convert("RGB")
692
 
693
- img = base64_to_image(b64_img)
694
- inputs = clip_proc(images=img, return_tensors="pt")
695
- with torch.no_grad():
696
- emb = clip_model.get_image_features(**inputs)
697
- emb = emb / emb.norm(p=2, dim=-1, keepdim=True)
698
- heatmap_emb = emb.squeeze().cpu().numpy()
699
-
700
- # === Step 6. 從 HF dataset 載入已儲存 heatmap embeddings ===
701
- from datasets import load_dataset
702
- ds = load_dataset(HF_HEATMAP_REPO, split="train")
703
-
704
- # 假設 Space A 已經知道當前樣本的 label(可從 model predict 帶進來)
705
- target_label = os.getenv("CURRENT_LABEL", "UNKNOWN")
706
- same_label = [x for x in ds if x["label"] == target_label]
707
- if not same_label:
708
- print(f"⚠️ No samples with label {target_label}, fallback to all data")
709
- same_label = ds
710
-
711
- # === Step 7. 相似度搜尋 ===
712
- dataset_vecs = np.stack(same_label["heatmap_embedding"])
713
- sims = cosine_similarity([heatmap_emb], dataset_vecs)[0]
714
- best_idx = int(np.argmax(sims))
715
- best_sample = same_label[best_idx]
716
- best_score = float(sims[best_idx])
717
-
718
- print(f"🎯 Most similar heatmap in label {target_label}: {best_sample['filename']} (sim={best_score:.3f})")
719
-
720
- # === Step 8. 回傳結果 ===
721
  return {
722
- "attention_heatmap": b64_img,
723
- "most_similar_in_label": best_sample["filename"],
724
- "similarity_score": best_score,
725
- "similar_heatmap_image": best_sample.get("heatmap_base64", None),
 
726
  }
727
 
728
  except Exception as e:
@@ -734,7 +528,6 @@ async def attention_heatmap(file: UploadFile = File(...)):
734
 
735
 
736
 
737
-
738
  @app.post("/check-new-samples")
739
  async def check_new_samples(background_tasks: BackgroundTasks):
740
  """
 
154
  return False
155
 
156
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
157
  @app.post("/predict")
158
  async def predict_malware(files: List[UploadFile] = File(...)):
159
  """
160
+ 批次預測惡意軟體家族(維持原回傳格式)
161
+ 並自動寫入 embeddings_log.jsonl、上傳到 Hugging Face Dataset
162
  """
163
  if not files:
164
  raise HTTPException(status_code=400, detail="No files provided")
165
 
166
+ print(f"\n{'='*60}")
167
+ print(f"📥 Received {len(files)} files")
168
+ print(f"{'='*60}")
169
+
170
  predictions = []
171
  all_embeddings = []
172
 
173
  try:
174
  with torch.no_grad():
175
+ for idx, file in enumerate(files, 1):
176
+ try:
177
+ content = await file.read()
178
+ asm_code = content.decode('utf-8').strip()
179
+ if not asm_code:
180
+ print(f"⚠️ Skip empty file: {file.filename}")
181
+ continue
182
+
183
+ print(f"📄 [{idx}/{len(files)}] Processing: {file.filename}")
184
+
185
+ # === Tokenize ===
186
+ inputs = tokenizer(
187
+ asm_code,
188
+ return_tensors="pt",
189
+ max_length=512,
190
+ truncation=True,
191
+ padding=True
192
+ )
193
+ inputs = {k: v.to(DEVICE) for k, v in inputs.items()}
194
+
195
+ # === Forward ===
196
+ outputs = model(**inputs)
197
+ logits = outputs.logits
198
+
199
+ # === Softmax 機率 ===
200
+ probs = torch.nn.functional.softmax(logits, dim=-1)
201
+ prob_list = probs.squeeze().cpu().tolist()
202
+
203
+ # === 取得預測結果 ===
204
+ confidence, predicted_class = torch.max(probs, dim=-1)
205
+ predicted_id = predicted_class.item()
206
+ label_name = (
207
+ model.config.id2label[predicted_id]
208
+ if hasattr(model.config, "id2label") and predicted_id in model.config.id2label
209
+ else f"class_{predicted_id}"
210
+ )
211
+ print(f" 🔍 Predicted ID: {predicted_id}, Label: {label_name}")
212
+
213
+ # === Embedding ===
214
+ if hasattr(outputs, "hidden_states") and outputs.hidden_states is not None:
215
+ last_hidden = outputs.hidden_states[-1]
216
+ cls_embedding = last_hidden[:, 0, :].cpu().numpy()[0]
217
+ else:
218
+ cls_embedding = logits.cpu().numpy()[0]
219
+
220
+ # === Attention ===
221
+ max_attention_score = 0.0
222
+ if hasattr(outputs, "attentions") and outputs.attentions is not None:
223
+ last_attention = outputs.attentions[-1]
224
+ avg_attention = last_attention.mean(dim=1)
225
+ cls_attention = avg_attention[0, 0, :].cpu().numpy()
226
+ max_attention_score = float(cls_attention.max())
227
+
228
+ # === Collect Results ===
229
+ predictions.append({
230
+ "filename": file.filename,
231
+ "predicted_label": label_name,
232
+ "confidence": float(confidence.item()),
233
+ "attention_score": max_attention_score
234
+ })
235
+
236
+ all_embeddings.append({
237
+ "source": file.filename,
238
+ "label": label_name,
239
+ "embedding": cls_embedding.tolist(),
240
+ "attention_score": max_attention_score,
241
+ "text_excerpt": asm_code,
242
+ "probs": prob_list # ✅ 新增:完整 softmax 機率
243
+ })
244
+
245
+ print(f" ✅ Predicted: {label_name} (confidence: {confidence.item():.3f})")
246
+
247
+ except Exception as e:
248
+ print(f" ❌ Error processing {file.filename}: {e}")
249
  continue
250
+
251
+ if not predictions:
252
+ raise HTTPException(status_code=400, detail="No valid files processed")
253
+
254
+ # === 多數投票 ===
255
+ label_votes = [p["predicted_label"] for p in predictions]
256
+ vote_counts = Counter(label_votes)
257
+ final_label, final_count = vote_counts.most_common(1)[0]
258
+
259
+ print(f"\n📊 Vote distribution: {dict(vote_counts)}")
260
+ print(f"🏆 Final prediction: {final_label} ({final_count}/{len(predictions)} votes)")
261
+
262
+ # === 選最高注意力權重的 embedding ===
263
+ same_label_embeddings = [emb for emb in all_embeddings if emb["label"] == final_label]
264
+ best_embedding = (
265
+ max(same_label_embeddings, key=lambda x: x["attention_score"])
266
+ if same_label_embeddings else
267
+ max(all_embeddings, key=lambda x: x["attention_score"])
268
+ )
269
+
270
+ print(f"🎯 Selected embedding from: {best_embedding['source']}")
271
+ print(f" Attention score: {best_embedding['attention_score']:.4f}")
272
+
273
+ # === 回傳結果 ===
 
 
 
 
 
 
 
 
 
 
 
 
274
  result = {
275
  "final_label": final_label,
276
+ "vote_count": final_count,
277
+ "total_segments": len(predictions),
278
+ "confidence": sum(p["confidence"] for p in predictions) / len(predictions),
279
+ "vote_distribution": dict(vote_counts),
280
+ "embedding": {
281
+ "values": best_embedding["embedding"],
282
+ "source_file": best_embedding["source"],
283
+ "attention_score": best_embedding["attention_score"],
284
+ "dimension": len(best_embedding["embedding"])
285
+ },
286
+ "segment_predictions": predictions
287
  }
288
 
289
+ print(f"\n✅ Prediction complete!\n")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
290
 
291
+
292
+ # === 寫入本地 embeddings_log.jsonl ===
293
  log_path = os.path.join("results", "embeddings_log.jsonl")
294
  with open(log_path, "a") as f:
295
  for emb in all_embeddings:
296
  f.write(json.dumps(emb) + "\n")
297
+ print(f"📝 Logged {len(all_embeddings)} embeddings locally.")
298
+
299
+ # === 上傳到 Hugging Face Dataset ===
300
+ repo_id = "lyi029/malvec-embeddings-new" # ✅ 你的新 dataset
301
+ api = HfApi()
302
+
303
+ try:
304
+ old_data = []
305
+ try:
306
+ old_path = hf_hub_download(
307
+ repo_id=repo_id,
308
+ filename="embeddings_log.jsonl",
309
+ repo_type="dataset",
310
+ token=HF_TOKEN
311
+ )
312
+ with open(old_path, "r") as f:
313
+ old_data = [json.loads(l) for l in f if l.strip()]
314
+ except Exception:
315
+ print("ℹ️ No previous embeddings found in dataset.")
316
+
317
+ with open(log_path, "r") as f:
318
+ new_data = [json.loads(l) for l in f if l.strip()]
319
+
320
+ # 合併去重
321
+ merged_dict = {(d["source"]): d for d in (old_data + new_data)}
322
+ merged = list(merged_dict.values())
323
+
324
+ with open(log_path, "w") as f:
325
+ for item in merged:
326
+ f.write(json.dumps(item) + "\n")
327
+
328
+ api.upload_file(
329
+ path_or_fileobj=log_path,
330
+ path_in_repo="embeddings_log.jsonl",
331
+ repo_id=repo_id,
332
+ repo_type="dataset",
333
+ token=HF_TOKEN,
334
+ )
335
+ print(f"☁️ Uploaded {len(merged)} merged embeddings to dataset: {repo_id}")
336
+ except Exception as e:
337
+ print(f"⚠️ Dataset upload failed: {e}")
338
 
339
  return result
340
 
341
  except Exception as e:
342
+ print(f"\n❌ Error in predict_malware: {e}")
343
  import traceback
344
  traceback.print_exc()
345
+ raise HTTPException(status_code=500, detail=f"Prediction failed: {str(e)}")
 
346
 
347
  csv.field_size_limit(sys.maxsize)
348
 
 
440
  # except Exception as e:
441
  # print(f"❌ attention_heatmap error: {e}")
442
  # return {"error": str(e)}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
443
  @app.post("/attention-heatmap")
444
  async def attention_heatmap(file: UploadFile = File(...)):
445
  """
446
  適用格式:整行以 '</s>' 分隔的 disasm tokens。
447
  每個 block 以 jump/call/ret 結束。
448
  最多取前 1000 個 block。
449
+ 會同時回傳:
450
+ - blocks:每個 basic block 的注意力平均
451
+ - attention_heatmap:base64 圖像
452
+ - heatmap_matrix:數值陣列(可供前端繪圖)
453
  """
454
+ import io, sys, numpy as np, csv, base64, matplotlib.pyplot as plt
 
 
 
 
 
455
  csv.field_size_limit(sys.maxsize)
 
456
 
457
  try:
458
+ # === 讀檔 ===
459
  content = await file.read()
460
  text = content.decode("utf-8", errors="ignore")
461
 
462
+ # === 以 </s> 分隔整個 disasm ===
463
  lines = text.replace("\n", " ").split("</s>")
464
  tokens = []
465
  for line in lines:
466
  toks = [t.strip() for t in line.split() if t.strip()]
467
  if toks:
468
  tokens.extend(toks)
 
469
 
470
+ print(f"📦 Total tokens parsed: {len(tokens)}")
471
+
472
+ # === 拆 basic blocks ===
473
+ blocks = _split_into_basic_blocks(tokens)
474
+ print(f"📊 Total blocks before truncation: {len(blocks)}")
475
+
476
+ # === 只取前 1000 個 block
477
+ MAX_BLOCKS = int(os.getenv("MAX_HEATMAP_BLOCKS", "1000"))
478
+ blocks = blocks[:MAX_BLOCKS]
479
+
480
+ # === 計算每個 block 的 attention
481
+ results = []
482
+ for i, blk in enumerate(blocks):
483
  score = _block_attention_score(blk)
484
+ results.append({
485
+ "idx": i,
486
+ "size": len(blk),
487
+ "score": score
488
+ })
489
+
490
+ if not results:
491
+ return {"error": "No valid blocks parsed"}
492
 
493
+ scores = [b["score"] for b in results]
494
+ avg_score = float(np.mean(scores))
495
+
496
+ # === 視覺化成 heatmap ===
497
  fig, ax = plt.subplots(figsize=(10, 2))
498
  heatmap_data = np.expand_dims(scores, axis=0)
499
  im = ax.imshow(heatmap_data, cmap="viridis", aspect="auto")
500
  plt.colorbar(im, ax=ax, orientation="vertical", label="Attention Score")
501
  ax.set_title("Attention Heatmap per Basic Block")
502
  ax.set_xlabel("Block Index")
503
+ ax.set_yticks([]) # 不需要 Y 軸
504
  plt.tight_layout()
505
 
506
+ # === 轉成 base64 ===
507
  buf = io.BytesIO()
508
  plt.savefig(buf, format="png")
509
  buf.seek(0)
510
  img_base64 = base64.b64encode(buf.read()).decode("utf-8")
511
  plt.close(fig)
 
 
 
 
 
 
 
 
 
512
 
513
+ # === 回傳 JSON ===
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
514
  return {
515
+ # "num_blocks": len(results),
516
+ # "avg_attention": avg_score,
517
+ # "blocks": results,
518
+ # "heatmap_matrix": heatmap_data.tolist(), # ✅ 可互動繪圖
519
+ "attention_heatmap": f"data:image/png;base64,{img_base64}" # ✅ 前端直接顯示
520
  }
521
 
522
  except Exception as e:
 
528
 
529
 
530
 
 
531
  @app.post("/check-new-samples")
532
  async def check_new_samples(background_tasks: BackgroundTasks):
533
  """