lyi029 commited on
Commit
7c704b6
·
1 Parent(s): b7d74eb
Files changed (1) hide show
  1. app.py +210 -20
app.py CHANGED
@@ -30,7 +30,7 @@ app.add_middleware(
30
  MODEL_REPO = os.getenv("MODEL_REPO", "lyi029/codebert-finetuned")
31
  HF_TOKEN = os.getenv("HF_TOKEN")
32
  DEVICE = torch.device("cpu")
33
- HF_DATASET_REPO = "lyi029/malvec-embeddings"
34
 
35
  # ====== 連接 Space B ======
36
  SPACE_B_URL = os.getenv("SPACE_B_URL", "https://lyi029-model-update.hf.space")
@@ -425,6 +425,193 @@ def upload_embeddings_to_hub(local_path: str):
425
 
426
  # except Exception as e:
427
  # raise HTTPException(status_code=500, detail=f"Prediction failed: {e}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
428
  @app.post("/predict")
429
  async def predict_malware(files: List[UploadFile] = File(...)):
430
  """
@@ -447,9 +634,8 @@ async def predict_malware(files: List[UploadFile] = File(...)):
447
  try:
448
  content = await file.read()
449
  asm_code = content.decode('utf-8').strip()
450
-
451
  if not asm_code:
452
- print(f"⚠️ Skip empty file: {file.filename}")
453
  continue
454
 
455
  print(f"📄 [{idx}/{len(files)}] Processing: {file.filename}")
@@ -467,16 +653,19 @@ async def predict_malware(files: List[UploadFile] = File(...)):
467
  # === Forward ===
468
  outputs = model(**inputs)
469
  logits = outputs.logits
 
 
470
  probs = torch.nn.functional.softmax(logits, dim=-1)
471
- confidence, predicted_class = torch.max(probs, dim=-1)
472
 
 
 
473
  predicted_id = predicted_class.item()
474
  label_name = (
475
  model.config.id2label[predicted_id]
476
  if hasattr(model.config, "id2label") and predicted_id in model.config.id2label
477
  else f"class_{predicted_id}"
478
  )
479
-
480
  print(f" 🔍 Predicted ID: {predicted_id}, Label: {label_name}")
481
 
482
  # === Embedding ===
@@ -503,11 +692,12 @@ async def predict_malware(files: List[UploadFile] = File(...)):
503
  })
504
 
505
  all_embeddings.append({
506
- "filename": file.filename,
507
  "label": label_name,
508
  "embedding": cls_embedding.tolist(),
509
  "attention_score": max_attention_score,
510
- "text_excerpt": asm_code[:2000] # 方便後續分析
 
511
  })
512
 
513
  print(f" ✅ Predicted: {label_name} (confidence: {confidence.item():.3f})")
@@ -522,20 +712,20 @@ async def predict_malware(files: List[UploadFile] = File(...)):
522
  # === 多數投票 ===
523
  label_votes = [p["predicted_label"] for p in predictions]
524
  vote_counts = Counter(label_votes)
525
- final_label = vote_counts.most_common(1)[0][0]
526
- final_count = vote_counts[final_label]
527
 
528
  print(f"\n📊 Vote distribution: {dict(vote_counts)}")
529
  print(f"🏆 Final prediction: {final_label} ({final_count}/{len(predictions)} votes)")
530
 
531
- # === 最高注意力權重的 embedding ===
532
  same_label_embeddings = [emb for emb in all_embeddings if emb["label"] == final_label]
533
- if same_label_embeddings:
534
- best_embedding = max(same_label_embeddings, key=lambda x: x["attention_score"])
535
- else:
536
- best_embedding = max(all_embeddings, key=lambda x: x["attention_score"])
 
537
 
538
- print(f"🎯 Selected embedding from: {best_embedding['filename']}")
539
  print(f" Attention score: {best_embedding['attention_score']:.4f}")
540
 
541
  # === 回傳結果 ===
@@ -547,7 +737,7 @@ async def predict_malware(files: List[UploadFile] = File(...)):
547
  "vote_distribution": dict(vote_counts),
548
  "embedding": {
549
  "values": best_embedding["embedding"],
550
- "source_file": best_embedding["filename"],
551
  "attention_score": best_embedding["attention_score"],
552
  "dimension": len(best_embedding["embedding"])
553
  },
@@ -557,7 +747,6 @@ async def predict_malware(files: List[UploadFile] = File(...)):
557
  print(f"\n✅ Prediction complete!\n")
558
 
559
  # === 寫入本地 embeddings_log.jsonl ===
560
- os.makedirs("results", exist_ok=True)
561
  log_path = os.path.join("results", "embeddings_log.jsonl")
562
  with open(log_path, "a") as f:
563
  for emb in all_embeddings:
@@ -565,7 +754,7 @@ async def predict_malware(files: List[UploadFile] = File(...)):
565
  print(f"📝 Logged {len(all_embeddings)} embeddings locally.")
566
 
567
  # === 上傳到 Hugging Face Dataset ===
568
- repo_id = "lyi029/malvec-embeddings"
569
  api = HfApi()
570
 
571
  try:
@@ -582,11 +771,11 @@ async def predict_malware(files: List[UploadFile] = File(...)):
582
  except Exception:
583
  print("ℹ️ No previous embeddings found in dataset.")
584
 
585
- # 合併去重
586
  with open(log_path, "r") as f:
587
  new_data = [json.loads(l) for l in f if l.strip()]
588
 
589
- merged_dict = {d["filename"]: d for d in (old_data + new_data)}
 
590
  merged = list(merged_dict.values())
591
 
592
  with open(log_path, "w") as f:
@@ -612,6 +801,7 @@ async def predict_malware(files: List[UploadFile] = File(...)):
612
  traceback.print_exc()
613
  raise HTTPException(status_code=500, detail=f"Prediction failed: {str(e)}")
614
 
 
615
  # ----------------------------------------------------------
616
  # 手動觸發 Space B 訓練
617
  # ----------------------------------------------------------
 
30
  MODEL_REPO = os.getenv("MODEL_REPO", "lyi029/codebert-finetuned")
31
  HF_TOKEN = os.getenv("HF_TOKEN")
32
  DEVICE = torch.device("cpu")
33
+ HF_DATASET_REPO = "lyi029/malvec-embeddings-new"
34
 
35
  # ====== 連接 Space B ======
36
  SPACE_B_URL = os.getenv("SPACE_B_URL", "https://lyi029-model-update.hf.space")
 
425
 
426
  # except Exception as e:
427
  # raise HTTPException(status_code=500, detail=f"Prediction failed: {e}")
428
+ # @app.post("/predict")
429
+ # async def predict_malware(files: List[UploadFile] = File(...)):
430
+ # """
431
+ # 批次預測惡意軟體家族(維持原回傳格式)
432
+ # 並自動寫入 embeddings_log.jsonl、上傳到 Hugging Face Dataset
433
+ # """
434
+ # if not files:
435
+ # raise HTTPException(status_code=400, detail="No files provided")
436
+
437
+ # print(f"\n{'='*60}")
438
+ # print(f"📥 Received {len(files)} files")
439
+ # print(f"{'='*60}")
440
+
441
+ # predictions = []
442
+ # all_embeddings = []
443
+
444
+ # try:
445
+ # with torch.no_grad():
446
+ # for idx, file in enumerate(files, 1):
447
+ # try:
448
+ # content = await file.read()
449
+ # asm_code = content.decode('utf-8').strip()
450
+
451
+ # if not asm_code:
452
+ # print(f"⚠️ Skip empty file: {file.filename}")
453
+ # continue
454
+
455
+ # print(f"📄 [{idx}/{len(files)}] Processing: {file.filename}")
456
+
457
+ # # === Tokenize ===
458
+ # inputs = tokenizer(
459
+ # asm_code,
460
+ # return_tensors="pt",
461
+ # max_length=512,
462
+ # truncation=True,
463
+ # padding=True
464
+ # )
465
+ # inputs = {k: v.to(DEVICE) for k, v in inputs.items()}
466
+
467
+ # # === Forward ===
468
+ # outputs = model(**inputs)
469
+ # logits = outputs.logits
470
+ # probs = torch.nn.functional.softmax(logits, dim=-1)
471
+ # confidence, predicted_class = torch.max(probs, dim=-1)
472
+
473
+ # predicted_id = predicted_class.item()
474
+ # label_name = (
475
+ # model.config.id2label[predicted_id]
476
+ # if hasattr(model.config, "id2label") and predicted_id in model.config.id2label
477
+ # else f"class_{predicted_id}"
478
+ # )
479
+
480
+ # print(f" 🔍 Predicted ID: {predicted_id}, Label: {label_name}")
481
+
482
+ # # === Embedding ===
483
+ # if hasattr(outputs, "hidden_states") and outputs.hidden_states is not None:
484
+ # last_hidden = outputs.hidden_states[-1]
485
+ # cls_embedding = last_hidden[:, 0, :].cpu().numpy()[0]
486
+ # else:
487
+ # cls_embedding = logits.cpu().numpy()[0]
488
+
489
+ # # === Attention ===
490
+ # max_attention_score = 0.0
491
+ # if hasattr(outputs, "attentions") and outputs.attentions is not None:
492
+ # last_attention = outputs.attentions[-1]
493
+ # avg_attention = last_attention.mean(dim=1)
494
+ # cls_attention = avg_attention[0, 0, :].cpu().numpy()
495
+ # max_attention_score = float(cls_attention.max())
496
+
497
+ # # === Collect Results ===
498
+ # predictions.append({
499
+ # "filename": file.filename,
500
+ # "predicted_label": label_name,
501
+ # "confidence": float(confidence.item()),
502
+ # "attention_score": max_attention_score
503
+ # })
504
+
505
+ # all_embeddings.append({
506
+ # "source": file.filename,
507
+ # "label": label_name,
508
+ # "embedding": cls_embedding.tolist(),
509
+ # "attention_score": max_attention_score,
510
+ # "text_excerpt": asm_code # 方便後續分析
511
+ # })
512
+
513
+ # print(f" ✅ Predicted: {label_name} (confidence: {confidence.item():.3f})")
514
+
515
+ # except Exception as e:
516
+ # print(f" ❌ Error processing {file.filename}: {e}")
517
+ # continue
518
+
519
+ # if not predictions:
520
+ # raise HTTPException(status_code=400, detail="No valid files processed")
521
+
522
+ # # === 多數投票 ===
523
+ # label_votes = [p["predicted_label"] for p in predictions]
524
+ # vote_counts = Counter(label_votes)
525
+ # final_label = vote_counts.most_common(1)[0][0]
526
+ # final_count = vote_counts[final_label]
527
+
528
+ # print(f"\n📊 Vote distribution: {dict(vote_counts)}")
529
+ # print(f"🏆 Final prediction: {final_label} ({final_count}/{len(predictions)} votes)")
530
+
531
+ # # === 最高注意力權重的 embedding ===
532
+ # same_label_embeddings = [emb for emb in all_embeddings if emb["label"] == final_label]
533
+ # if same_label_embeddings:
534
+ # best_embedding = max(same_label_embeddings, key=lambda x: x["attention_score"])
535
+ # else:
536
+ # best_embedding = max(all_embeddings, key=lambda x: x["attention_score"])
537
+
538
+ # print(f"🎯 Selected embedding from: {best_embedding['filename']}")
539
+ # print(f" Attention score: {best_embedding['attention_score']:.4f}")
540
+
541
+ # # === 回傳結果 ===
542
+ # result = {
543
+ # "final_label": final_label,
544
+ # "vote_count": final_count,
545
+ # "total_segments": len(predictions),
546
+ # "confidence": sum(p["confidence"] for p in predictions) / len(predictions),
547
+ # "vote_distribution": dict(vote_counts),
548
+ # "embedding": {
549
+ # "values": best_embedding["embedding"],
550
+ # "source_file": best_embedding["filename"],
551
+ # "attention_score": best_embedding["attention_score"],
552
+ # "dimension": len(best_embedding["embedding"])
553
+ # },
554
+ # "segment_predictions": predictions
555
+ # }
556
+
557
+ # print(f"\n✅ Prediction complete!\n")
558
+
559
+ # # === 寫入本地 embeddings_log.jsonl ===
560
+ # os.makedirs("results", exist_ok=True)
561
+ # log_path = os.path.join("results", "embeddings_log.jsonl")
562
+ # with open(log_path, "a") as f:
563
+ # for emb in all_embeddings:
564
+ # f.write(json.dumps(emb) + "\n")
565
+ # print(f"📝 Logged {len(all_embeddings)} embeddings locally.")
566
+
567
+ # # === 上傳到 Hugging Face Dataset ===
568
+ # repo_id = "lyi029/malvec-embeddings"
569
+ # api = HfApi()
570
+
571
+ # try:
572
+ # old_data = []
573
+ # try:
574
+ # old_path = hf_hub_download(
575
+ # repo_id=repo_id,
576
+ # filename="embeddings_log.jsonl",
577
+ # repo_type="dataset",
578
+ # token=HF_TOKEN
579
+ # )
580
+ # with open(old_path, "r") as f:
581
+ # old_data = [json.loads(l) for l in f if l.strip()]
582
+ # except Exception:
583
+ # print("ℹ️ No previous embeddings found in dataset.")
584
+
585
+ # # 合併去重
586
+ # with open(log_path, "r") as f:
587
+ # new_data = [json.loads(l) for l in f if l.strip()]
588
+
589
+ # merged_dict = {d["filename"]: d for d in (old_data + new_data)}
590
+ # merged = list(merged_dict.values())
591
+
592
+ # with open(log_path, "w") as f:
593
+ # for item in merged:
594
+ # f.write(json.dumps(item) + "\n")
595
+
596
+ # api.upload_file(
597
+ # path_or_fileobj=log_path,
598
+ # path_in_repo="embeddings_log.jsonl",
599
+ # repo_id=repo_id,
600
+ # repo_type="dataset",
601
+ # token=HF_TOKEN,
602
+ # )
603
+ # print(f"☁️ Uploaded {len(merged)} merged embeddings to dataset: {repo_id}")
604
+ # except Exception as e:
605
+ # print(f"⚠️ Dataset upload failed: {e}")
606
+
607
+ # return result
608
+
609
+ # except Exception as e:
610
+ # print(f"\n❌ Error in predict_malware: {e}")
611
+ # import traceback
612
+ # traceback.print_exc()
613
+ # raise HTTPException(status_code=500, detail=f"Prediction failed: {str(e)}")
614
+
615
  @app.post("/predict")
616
  async def predict_malware(files: List[UploadFile] = File(...)):
617
  """
 
634
  try:
635
  content = await file.read()
636
  asm_code = content.decode('utf-8').strip()
 
637
  if not asm_code:
638
+ print(f"⚠️ Skip empty file: {file.filename}")
639
  continue
640
 
641
  print(f"📄 [{idx}/{len(files)}] Processing: {file.filename}")
 
653
  # === Forward ===
654
  outputs = model(**inputs)
655
  logits = outputs.logits
656
+
657
+ # === Softmax 機率 ===
658
  probs = torch.nn.functional.softmax(logits, dim=-1)
659
+ prob_list = probs.squeeze().cpu().tolist()
660
 
661
+ # === 取得預測結果 ===
662
+ confidence, predicted_class = torch.max(probs, dim=-1)
663
  predicted_id = predicted_class.item()
664
  label_name = (
665
  model.config.id2label[predicted_id]
666
  if hasattr(model.config, "id2label") and predicted_id in model.config.id2label
667
  else f"class_{predicted_id}"
668
  )
 
669
  print(f" 🔍 Predicted ID: {predicted_id}, Label: {label_name}")
670
 
671
  # === Embedding ===
 
692
  })
693
 
694
  all_embeddings.append({
695
+ "source": file.filename,
696
  "label": label_name,
697
  "embedding": cls_embedding.tolist(),
698
  "attention_score": max_attention_score,
699
+ "text_excerpt": asm_code,
700
+ "probs": prob_list # ✅ 新增:完整 softmax 機率
701
  })
702
 
703
  print(f" ✅ Predicted: {label_name} (confidence: {confidence.item():.3f})")
 
712
  # === 多數投票 ===
713
  label_votes = [p["predicted_label"] for p in predictions]
714
  vote_counts = Counter(label_votes)
715
+ final_label, final_count = vote_counts.most_common(1)[0]
 
716
 
717
  print(f"\n📊 Vote distribution: {dict(vote_counts)}")
718
  print(f"🏆 Final prediction: {final_label} ({final_count}/{len(predictions)} votes)")
719
 
720
+ # === 選最高注意力權重的 embedding ===
721
  same_label_embeddings = [emb for emb in all_embeddings if emb["label"] == final_label]
722
+ best_embedding = (
723
+ max(same_label_embeddings, key=lambda x: x["attention_score"])
724
+ if same_label_embeddings else
725
+ max(all_embeddings, key=lambda x: x["attention_score"])
726
+ )
727
 
728
+ print(f"🎯 Selected embedding from: {best_embedding['source']}")
729
  print(f" Attention score: {best_embedding['attention_score']:.4f}")
730
 
731
  # === 回傳結果 ===
 
737
  "vote_distribution": dict(vote_counts),
738
  "embedding": {
739
  "values": best_embedding["embedding"],
740
+ "source_file": best_embedding["source"],
741
  "attention_score": best_embedding["attention_score"],
742
  "dimension": len(best_embedding["embedding"])
743
  },
 
747
  print(f"\n✅ Prediction complete!\n")
748
 
749
  # === 寫入本地 embeddings_log.jsonl ===
 
750
  log_path = os.path.join("results", "embeddings_log.jsonl")
751
  with open(log_path, "a") as f:
752
  for emb in all_embeddings:
 
754
  print(f"📝 Logged {len(all_embeddings)} embeddings locally.")
755
 
756
  # === 上傳到 Hugging Face Dataset ===
757
+ repo_id = "lyi029/malvec-embeddings-new" # ✅ 你的新 dataset
758
  api = HfApi()
759
 
760
  try:
 
771
  except Exception:
772
  print("ℹ️ No previous embeddings found in dataset.")
773
 
 
774
  with open(log_path, "r") as f:
775
  new_data = [json.loads(l) for l in f if l.strip()]
776
 
777
+ # 合併去重
778
+ merged_dict = {(d["source"]): d for d in (old_data + new_data)}
779
  merged = list(merged_dict.values())
780
 
781
  with open(log_path, "w") as f:
 
801
  traceback.print_exc()
802
  raise HTTPException(status_code=500, detail=f"Prediction failed: {str(e)}")
803
 
804
+
805
  # ----------------------------------------------------------
806
  # 手動觸發 Space B 訓練
807
  # ----------------------------------------------------------