raxhel commited on
Commit
4be45cc
·
verified ·
1 Parent(s): 5dfe2df

Update app.py

Browse files

add som to app.py

Files changed (1) hide show
  1. app.py +208 -7
app.py CHANGED
@@ -5,7 +5,7 @@ import torch
5
  from fastapi import FastAPI, UploadFile, File, HTTPException, BackgroundTasks
6
  from fastapi.middleware.cors import CORSMiddleware
7
  from transformers import AutoTokenizer, AutoModelForSequenceClassification
8
- from typing import List
9
  from collections import Counter
10
  from fastapi.staticfiles import StaticFiles
11
  from huggingface_hub import HfApi,hf_hub_download
@@ -13,8 +13,19 @@ import datetime
13
  import csv, sys
14
  import io
15
  import numpy as np
16
-
17
-
 
 
 
 
 
 
 
 
 
 
 
18
  CURRENT_TAG = os.getenv("MODEL_TAG", "main")
19
  # ====== FastAPI 初始化 ======
20
  app = FastAPI(
@@ -67,6 +78,36 @@ model.to(DEVICE).eval()
67
  print("✅ Model loaded successfully")
68
 
69
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
70
  # ----------------------------------------------------------
71
  # ROUTES
72
  # ----------------------------------------------------------
@@ -74,13 +115,16 @@ print("✅ Model loaded successfully")
74
  @app.get("/")
75
  def home():
76
  return {
77
- "message": "Malware Classifier API (Space A)",
78
  "model": MODEL_REPO,
79
  "space_b_url": SPACE_B_URL,
 
80
  "endpoints": {
81
- "/predict": "POST - Upload TXT files for prediction",
 
82
  "/model-info": "GET - Get model information",
83
- "/trigger-train": "POST - Manually trigger Space B LoRA training"
 
84
  }
85
  }
86
 
@@ -103,7 +147,9 @@ def get_model_info():
103
  "labels": list(model.config.id2label.values())
104
  if hasattr(model.config, 'id2label') else None,
105
  "device": str(DEVICE),
106
- "id2label_sample": id2label_sample
 
 
107
  }
108
 
109
 
@@ -154,6 +200,125 @@ def upload_embeddings_to_hub(local_path: str):
154
  return False
155
 
156
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
157
  @app.post("/predict")
158
  async def predict_malware(files: List[UploadFile] = File(...)):
159
  """
@@ -337,6 +502,26 @@ async def predict_malware(files: List[UploadFile] = File(...)):
337
  except Exception as e:
338
  print(f"⚠️ Dataset upload failed: {e}")
339
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
340
  return result
341
 
342
  except Exception as e:
@@ -344,7 +529,23 @@ async def predict_malware(files: List[UploadFile] = File(...)):
344
  import traceback
345
  traceback.print_exc()
346
  raise HTTPException(status_code=500, detail=f"Prediction failed: {str(e)}")
 
 
 
 
 
 
 
 
 
 
347
 
 
 
 
 
 
 
348
  csv.field_size_limit(sys.maxsize)
349
 
350
  JUMP_MNEMONICS = {"jmp", "je", "jne", "jl", "jg", "jle", "jge", "call", "ret"}
 
5
  from fastapi import FastAPI, UploadFile, File, HTTPException, BackgroundTasks
6
  from fastapi.middleware.cors import CORSMiddleware
7
  from transformers import AutoTokenizer, AutoModelForSequenceClassification
8
+ from typing import List, Optional, Dict
9
  from collections import Counter
10
  from fastapi.staticfiles import StaticFiles
11
  from huggingface_hub import HfApi,hf_hub_download
 
13
  import csv, sys
14
  import io
15
  import numpy as np
16
+ from datasets import load_dataset
17
+
18
+ # ===== 新增:SOM 相關 import =====
19
+ from datasets import load_dataset
20
+
21
+ # ⚠️ 注意: som_analyzer.py 必須存在於同一目錄
22
+ try:
23
+ from som_analyzer import SOMAnalyzer
24
+ SOM_FILE_AVAILABLE = True
25
+ except ImportError:
26
+ print("⚠️ WARNING: 'som_analyzer.py' not found. SOM features will be disabled.")
27
+ SOM_FILE_AVAILABLE = False
28
+
29
  CURRENT_TAG = os.getenv("MODEL_TAG", "main")
30
  # ====== FastAPI 初始化 ======
31
  app = FastAPI(
 
78
  print("✅ Model loaded successfully")
79
 
80
 
81
+ # ===== 新增:載入 SOM 模型 (from app.py) =====
82
+ print("🧠 Initializing SOM Analyzer...")
83
+ SOM_REPO = "malvec/codebert_Malvec" # (app.py 的 SOM repo)
84
+ SOM_AVAILABLE = False
85
+
86
+ if SOM_FILE_AVAILABLE: # 檢查 'som_analyzer.py' 是否 import 成功
87
+ try:
88
+ som_weights_path = hf_hub_download(
89
+ repo_id=SOM_REPO,
90
+ filename="som_weights.npy",
91
+ token=HF_TOKEN
92
+ )
93
+ background_data_path = hf_hub_download(
94
+ repo_id=SOM_REPO,
95
+ filename="som_background_data.pkl",
96
+ token=HF_TOKEN
97
+ )
98
+
99
+ som_analyzer = SOMAnalyzer(
100
+ som_weights_path=som_weights_path,
101
+ background_data_path=background_data_path
102
+ )
103
+ print("✅ SOM Analyzer initialized successfully!")
104
+ SOM_AVAILABLE = True
105
+ except Exception as e:
106
+ print(f"⚠️ SOM Analyzer initialization failed: {e}")
107
+ print(" SOM analysis will be disabled")
108
+ else:
109
+ print(" SOM analysis disabled because 'som_analyzer.py' is missing.")
110
+
111
  # ----------------------------------------------------------
112
  # ROUTES
113
  # ----------------------------------------------------------
 
115
  @app.get("/")
116
  def home():
117
  return {
118
+ "message": "Malware Classifier API (Space A) with SOM", # ✨
119
  "model": MODEL_REPO,
120
  "space_b_url": SPACE_B_URL,
121
+ "som_status": "Available" if SOM_AVAILABLE else "Disabled", # ✨
122
  "endpoints": {
123
+ "/predict": "POST - Upload TXT files for prediction (with SOM)",
124
+ "/predict-batch": "POST - Simplified batch prediction (from app.py)",
125
  "/model-info": "GET - Get model information",
126
+ "/attention-heatmap": "POST - Generate attention heatmap (with CLIP)",
127
+ "/check-new-samples": "POST - Trigger Space B Active Learning"
128
  }
129
  }
130
 
 
147
  "labels": list(model.config.id2label.values())
148
  if hasattr(model.config, 'id2label') else None,
149
  "device": str(DEVICE),
150
+ "id2label_sample": id2label_sample,
151
+ "som_model_repo": SOM_REPO, # ✨
152
+ "som_available": SOM_AVAILABLE # ✨
153
  }
154
 
155
 
 
200
  return False
201
 
202
 
203
+ # ----------------------------------------------------------
204
+ # 新增:SOM 輔助函數 (from app.py)
205
+ # ----------------------------------------------------------
206
+
207
+ async def get_embeddings_from_hf_dataset(filename: str) -> Optional[List[np.ndarray]]:
208
+ """ (from app.py) """
209
+ try:
210
+ print(f"\n📦 Fetching embeddings from HF dataset for: {filename}")
211
+ # 注意: 這裡使用的是 app.py 中定義的 dataset (lyi029/malvec-embeddings)
212
+ dataset = load_dataset("lyi029/malvec-embeddings", split="train", streaming=True)
213
+ embeddings = []
214
+ count = 0
215
+ for sample in dataset:
216
+ sample_filename = sample.get('source', '') or sample.get('filename', '')
217
+ if filename.lower() in sample_filename.lower():
218
+ emb = sample.get('embedding') or sample.get('emb')
219
+ if emb is not None:
220
+ embeddings.append(np.array(emb, dtype=np.float32))
221
+ count += 1
222
+ if count >= 1000: break
223
+ if embeddings:
224
+ print(f"✅ Found {len(embeddings)} embeddings for {filename}")
225
+ return embeddings
226
+ else:
227
+ print(f"⚠️ No embeddings found for {filename} in dataset")
228
+ return None
229
+ except Exception as e:
230
+ print(f"❌ Error fetching from HF dataset: {e}")
231
+ return None
232
+
233
+ def analyze_som_region_probability(
234
+ som_analyzer: SOMAnalyzer,
235
+ center_row: int,
236
+ center_col: int,
237
+ embeddings: List[np.ndarray],
238
+ radius: int = 2
239
+ ) -> Dict[str, float]:
240
+ """ (from app.py) """
241
+ print(f" 🔍 Analyzing region around ({center_row}, {center_col}) with radius={radius}")
242
+ region_samples = []
243
+ for emb in embeddings:
244
+ sample_row, sample_col = som_analyzer.som.winner(emb) #
245
+ if abs(sample_row - center_row) <= radius and abs(sample_col - center_col) <= radius:
246
+ region_samples.append(emb)
247
+ print(f" Found {len(region_samples)} samples in region")
248
+ if len(region_samples) == 0:
249
+ return {"dropper": 0.0, "APT30": 0.0}
250
+
251
+ feature_counts = {"dropper": [], "spreader": [], "APT30": [], "Codoso_Gh0st_1": []}
252
+ if hasattr(som_analyzer, 'background_embeddings') and hasattr(som_analyzer, 'background_labels'):
253
+ for i, bg_emb in enumerate(som_analyzer.background_embeddings):
254
+ bg_row, bg_col = som_analyzer.som.winner(bg_emb) #
255
+ if abs(bg_row - center_row) <= radius and abs(bg_col - center_col) <= radius:
256
+ for feature_name in feature_counts.keys():
257
+ if feature_name in som_analyzer.background_labels:
258
+ feature_counts[feature_name].append(
259
+ som_analyzer.background_labels[feature_name][i]
260
+ )
261
+ probabilities = {}
262
+ for feature_name, values in feature_counts.items():
263
+ if len(values) > 0:
264
+ prob = sum(values) / len(values)
265
+ probabilities[feature_name] = float(prob)
266
+ else:
267
+ probabilities[feature_name] = 0.0
268
+ return probabilities
269
+
270
+
271
+ async def analyze_with_som(embedding, predicted_family, filename):
272
+ """ (from app.py) """
273
+ if not SOM_AVAILABLE:
274
+ print("⚠️ SOM analysis disabled")
275
+ return None
276
+ try:
277
+ print("\n🗺️ Performing SOM analysis...")
278
+ if isinstance(embedding, list):
279
+ embedding_array = np.array(embedding, dtype=np.float32)
280
+ else:
281
+ embedding_array = embedding
282
+
283
+ winner_row, winner_col = som_analyzer.find_winner_neuron(embedding_array)
284
+ print(f" 📍 Winner neuron: ({winner_row}, {winner_col})")
285
+
286
+ hf_embeddings = await get_embeddings_from_hf_dataset(filename)
287
+
288
+ if hf_embeddings and len(hf_embeddings) > 0:
289
+ print(f"✅ Using {len(hf_embeddings)} embeddings from HF dataset")
290
+ features_prob = analyze_som_region_probability( #
291
+ som_analyzer, winner_row, winner_col, hf_embeddings, radius=2
292
+ )
293
+ data_source = "hf_dataset"
294
+ else:
295
+ print(f"⚠️ No HF data found, using background data")
296
+ features_dict = som_analyzer.predict_features_from_position(
297
+ winner_row, winner_col, radius=1
298
+ )
299
+ features_prob = {k: float(v) for k, v in features_dict.items()}
300
+ data_source = "background_data"
301
+
302
+ features_binary = {
303
+ feature: 1 if prob > 0.2 else 0
304
+ for feature, prob in features_prob.items()
305
+ }
306
+ print(f" ✅ Features detected: {features_binary}")
307
+
308
+ result = {
309
+ # 🐛 (from app.py) Bug Fix: col_winner -> winner_col
310
+ "winner_position": {"row": int(winner_row), "col": int(winner_col)},
311
+ "features": features_binary,
312
+ "features_probability": features_prob,
313
+ "data_source": data_source
314
+ }
315
+ print(f"✅ SOM analysis complete!")
316
+ return result
317
+ except Exception as e:
318
+ print(f"❌ SOM analysis error: {e}")
319
+ return None
320
+
321
+
322
  @app.post("/predict")
323
  async def predict_malware(files: List[UploadFile] = File(...)):
324
  """
 
502
  except Exception as e:
503
  print(f"⚠️ Dataset upload failed: {e}")
504
 
505
+ # ===== 新增:執行 SOM 分析 (from app.py) =====
506
+ som_analysis = None
507
+ final_embedding_values = result.get("embedding", {}).get("values")
508
+
509
+ if SOM_AVAILABLE and final_embedding_values:
510
+ print("\n🗺️ Performing SOM analysis...")
511
+ try:
512
+ # 假設 SOM 分析需要第一個檔案的名稱作為參考
513
+ som_analysis = await analyze_with_som(
514
+ embedding=final_embedding_values,
515
+ predicted_family=result.get("final_label", "Unknown"),
516
+ filename=files[0].filename
517
+ )
518
+ except Exception as e:
519
+ print(f"❌ SOM analysis error: {e}")
520
+
521
+ # 將 SOM 結果附加到最終回傳
522
+ result["som_analysis"] = som_analysis
523
+ print(f"✅ SOM analysis complete.")
524
+
525
  return result
526
 
527
  except Exception as e:
 
529
  import traceback
530
  traceback.print_exc()
531
  raise HTTPException(status_code=500, detail=f"Prediction failed: {str(e)}")
532
+
533
+ # ===== 新增:/predict-batch 路由 (from app.py) =====
534
+ @app.post("/predict-batch")
535
+ async def predict_batch(files: List[UploadFile] = File(...)):
536
+ """
537
+ (from app.py)
538
+ 簡化版: 只返回 label 和 embedding,不包含詳細資訊
539
+ """
540
+ # 這將調用上面被修改過的 predict_malware 函數
541
+ result = await predict_malware(files)
542
 
543
+ return {
544
+ "final_label": result["final_label"],
545
+ "embedding": result["embedding"]["values"]
546
+ # SOM results are not included here, matching app.py's behavior
547
+ }
548
+
549
  csv.field_size_limit(sys.maxsize)
550
 
551
  JUMP_MNEMONICS = {"jmp", "je", "jne", "jl", "jg", "jle", "jge", "call", "ret"}